{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "modified at 090417\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Configurations:\n",
      "BACKBONE                       resnet50\n",
      "BACKBONE_STRIDES               [4, 8, 16, 32, 64]\n",
      "BATCH_SIZE                     8\n",
      "BBOX_STD_DEV                   [0.1 0.1 0.2 0.2]\n",
      "COMPUTE_BACKBONE_SHAPE         None\n",
      "DETECTION_MAX_INSTANCES        100\n",
      "DETECTION_MIN_CONFIDENCE       0.7\n",
      "DETECTION_NMS_THRESHOLD        0.3\n",
      "FPN_CLASSIF_FC_LAYERS_SIZE     1024\n",
      "GPU_COUNT                      1\n",
      "GRADIENT_CLIP_NORM             5.0\n",
      "IMAGES_PER_GPU                 8\n",
      "IMAGE_MAX_DIM                  256\n",
      "IMAGE_META_SIZE                16\n",
      "IMAGE_MIN_DIM                  256\n",
      "IMAGE_MIN_SCALE                0\n",
      "IMAGE_RESIZE_MODE              square\n",
      "IMAGE_SHAPE                    [256 256   3]\n",
      "LEARNING_MOMENTUM              0.9\n",
      "LEARNING_RATE                  0.001\n",
      "LOSS_WEIGHTS                   {'mrcnn_mask_loss': 1.0, 'rpn_bbox_loss': 1.0, 'rpn_class_loss': 2.0, 'mrcnn_class_loss': 2.0, 'mrcnn_bbox_loss': 1.0}\n",
      "MASK_POOL_SIZE                 14\n",
      "MASK_SHAPE                     [28, 28]\n",
      "MAX_GT_INSTANCES               100\n",
      "MEAN_PIXEL                     [123.7 116.8 103.9]\n",
      "MINI_MASK_SHAPE                (28, 28)\n",
      "NAME                           chips\n",
      "NUM_CLASSES                    4\n",
      "POOL_SIZE                      7\n",
      "POST_NMS_ROIS_INFERENCE        1000\n",
      "POST_NMS_ROIS_TRAINING         2000\n",
      "ROI_POSITIVE_RATIO             0.16\n",
      "RPN_ANCHOR_RATIOS              [0.5, 1, 2]\n",
      "RPN_ANCHOR_SCALES              (4, 8, 16, 32, 64)\n",
      "RPN_ANCHOR_STRIDE              1\n",
      "RPN_BBOX_STD_DEV               [0.1 0.1 0.2 0.2]\n",
      "RPN_NMS_THRESHOLD              0.7\n",
      "RPN_TRAIN_ANCHORS_PER_IMAGE    256\n",
      "STEPS_PER_EPOCH                1000\n",
      "TOP_DOWN_PYRAMID_SIZE          256\n",
      "TRAIN_BN                       False\n",
      "TRAIN_ROIS_PER_IMAGE           16\n",
      "USE_MINI_MASK                  True\n",
      "USE_RPN_ROIS                   True\n",
      "VALIDATION_STEPS               20\n",
      "WEIGHT_DECAY                   0.0001\n",
      "\n",
      "\n",
      "**** info of test dataset : ***  \n",
      "  ratio of null mask images:17 %\n",
      "  max flaw size: 208\n",
      "  min flaw size: 2\n",
      "  mean flaw size: 34.638380809595205\n",
      "  bump ratio:23 %\n",
      "  dent ratio:35 %\n",
      "  dot ratio:23 %\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import random\n",
    "import math\n",
    "import re\n",
    "import time\n",
    "import numpy as np\n",
    "import cv2\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import h5py\n",
    "import datetime\n",
    "\n",
    "# Root directory of the project\n",
    "path = '/mnt/sh_flex_storage/malu/venv/CHIPS_MRCNN'\n",
    "ROOT_DIR = os.path.abspath(path)\n",
    "\n",
    "# Import Mask RCNN\n",
    "sys.path.append(ROOT_DIR)  # To find local version of the library\n",
    "from mrcnn.config import Config\n",
    "from mrcnn import utils\n",
    "import mrcnn.model as modellib\n",
    "from mrcnn import visualize\n",
    "from mrcnn.model import log\n",
    "\n",
    "# Directory to save logs and trained model\n",
    "MODEL_DIR = os.path.join(ROOT_DIR, \"logs_chips4\")\n",
    "\n",
    "#train_path = '/mnt/sh_flex_storage/project/xcos_mask/data/trainset.h5'\n",
    "#train_set = h5py.File(train_path,'r')\n",
    "#train_images = train_set['input']\n",
    "#train_masks = train_set['output']\n",
    "#del train_set\n",
    "#NUM_TRAIN = train_images.shape[0] # 224*224\n",
    "\n",
    "test_path = '/mnt/sh_flex_storage/malu/venv/multilayer/datasets/data6_data.h5'\n",
    "test_set = h5py.File(test_path,'r')\n",
    "test_images = test_set['input']\n",
    "test_masks = test_set['output']\n",
    "del test_set\n",
    "NUM_TEST = test_images.shape[0]\n",
    "\n",
    "\n",
    "class ChipsConfig(Config):\n",
    "    \"\"\"Configuration for training on the toy shapes dataset.\n",
    "    Derives from the base Config class and overrides values specific\n",
    "    to the toy shapes dataset.\n",
    "    \"\"\"\n",
    "    # Give the configuration a recognizable name\n",
    "    NAME = \"chips\"\n",
    "\n",
    "    # CH\n",
    "    BACKBONE = \"resnet50\"\n",
    "    MINI_MASK_SHAPE = (28, 28)\n",
    "    DETECTION_MAX_INSTANCES = 100\n",
    "    ROI_POSITIVE_RATIO = 0.16\n",
    "    USE_MINI_MASK = True\n",
    "\n",
    "    # TRY CH\n",
    "    POOL_SIZE = 7\n",
    "    MASK_POOL_SIZE = 14\n",
    "\n",
    "    # Train on 1 GPU and 8 images per GPU. We can put multiple images on each\n",
    "    # GPU because the images are small. Batch size is 8 (GPUs * images/GPU).\n",
    "    GPU_COUNT = 1  # CH\n",
    "    IMAGES_PER_GPU = 8\n",
    "\n",
    "    # Number of classes (including background)\n",
    "    NUM_CLASSES = 1 + 3  # background + 3 shapes\n",
    "\n",
    "    # Use small images for faster training. Set the limits of the small side\n",
    "    # the large side, and that determines the image shape.\n",
    "    IMAGE_MIN_DIM = 256\n",
    "    IMAGE_MAX_DIM = 256\n",
    "\n",
    "    # Number of training and validation steps per epoch\n",
    "    #STEPS_PER_EPOCH = NUM_TRAIN // IMAGES_PER_GPU\n",
    "    VALIDATION_STEPS = 20\n",
    "\n",
    "    # Use smaller anchors because our image and objects are small\n",
    "    RPN_ANCHOR_SCALES = (4, 8, 16, 32, 64)  # anchor side in pixels # CH\n",
    "\n",
    "    # Reduce training ROIs per image because the images are small and have\n",
    "    # few objects. Aim to allow ROI sampling to pick 33% positive ROIs.\n",
    "    TRAIN_ROIS_PER_IMAGE = 16  # CH\n",
    "\n",
    "\n",
    "config = ChipsConfig()\n",
    "config.display()\n",
    "class ChipsDataset(utils.Dataset):\n",
    "    def __init__(self, class_map=None):\n",
    "        #new_append \n",
    "        self.num_null_mask = 0\n",
    "        self.all_flaw_size = []\n",
    "        self.subset = \"\"\n",
    "        #original\n",
    "        self._image_ids = []\n",
    "        self.image_info = []\n",
    "        # Background is always the first class\n",
    "        self.class_info = [{\"source\": \"\", \"id\": 0, \"name\": \"BG\"}]\n",
    "        self.source_class_ids = {}\n",
    "\n",
    "    def load_chips(self, count,subset):\n",
    "        assert subset in [\"train\",\"test\"]\n",
    "        \n",
    "        self.subset = subset\n",
    "        \n",
    "        # Add classes\n",
    "        self.add_class(\"chips\", 1, \"bump\")\n",
    "        self.add_class(\"chips\", 2, \"dent\")\n",
    "        self.add_class(\"chips\", 3, \"dot\")\n",
    "\n",
    "        # Add images\n",
    "        # Images are generated on the fly in load_image().\n",
    "        for i in range(count):\n",
    "            chips = self.mask_prepare(i)\n",
    "            self.add_image(\"chips\", image_id=i, path=None,chips=chips)\n",
    "            \n",
    "    \n",
    "    def mask_prepare(self,i,return_mask=False):\n",
    "        if self.subset == \"train\":\n",
    "            mask_full = train_masks[i,:].reshape((4,224,224))\n",
    "        else :\n",
    "            mask_full = test_masks[i,:].reshape((4,224,224))\n",
    "        \n",
    "        bump,dent,dot = mask_full[1,:,:],mask_full[2,:,:],mask_full[3,:,:]   \n",
    "        flaw_size = [np.sum(bump),np.sum(dent),np.sum(dot)]\n",
    "        names = [\"bump\",\"dent\",\"dot\"]\n",
    "\n",
    "        if not np.sum(flaw_size):\n",
    "            self.num_null_mask += 1\n",
    "            #print(\"==== no flaw for image %d ====\"%i)\n",
    "\n",
    "        if not return_mask:\n",
    "            chips= []\n",
    "            for j in range(3):\n",
    "                if flaw_size[j] >0:\n",
    "                    chips.append(names[j])\n",
    "                    self.all_flaw_size.append(flaw_size[j])\n",
    "            return chips\n",
    "        else:\n",
    "            masks = []\n",
    "            for j in range(3):\n",
    "                if flaw_size[j] >0:\n",
    "                    masks.append(mask_full[j+1,:,:])\n",
    "            if not np.array(masks).shape[0]:\n",
    "                return np.array(masks)\n",
    "            else:\n",
    "                masks = np.array(masks)\n",
    "                masks = masks.reshape((-1,224,224))\n",
    "                masks_reshape = np.zeros((224,224,masks.shape[0]))\n",
    "                #assert masks.shape[1:] in [(224,224)]\n",
    "                for i in range(224):\n",
    "                    for j in range(224):\n",
    "                        masks_reshape[i,j,:] = masks[:,i,j]  \n",
    "                return masks_reshape\n",
    "                      \n",
    "        \n",
    "    def load_image(self,image_id):\n",
    "        info = self.image_info[image_id]\n",
    "        \n",
    "        #print(\"check load_image image_id: \",image_id)\n",
    "        \n",
    "        if self.subset == \"train\":\n",
    "            image_full = train_images[image_id,:].reshape((6,224,224))\n",
    "        else:\n",
    "            image_full = test_images[image_id,:].reshape((6,224,224))\n",
    "            \n",
    "        image_sel = np.array([image_full[0,:,:],image_full[1,:,:],image_full[2,:,:]])\n",
    "        image_sel_reshape = np.zeros((224,224,3))\n",
    "        for i in range(224):\n",
    "            for j in range(224):\n",
    "                image_sel_reshape[i,j,:] = image_sel[:,i,j]  \n",
    "        image = image_sel_reshape\n",
    "        return image\n",
    "\n",
    "    def image_reference(self, image_id):\n",
    "        info = self.image_info[image_id]\n",
    "        if info[\"source\"] == \"chips\":\n",
    "            return info[\"chips\"]\n",
    "        else:\n",
    "            super(self.__class__).image_reference(self, image_id)\n",
    "\n",
    "    def load_mask(self, image_id):\n",
    "        \"\"\"Generate instance masks for shapes of the given image ID.\n",
    "        \"\"\"\n",
    "        info = self.image_info[image_id]\n",
    "        chips = info['chips']\n",
    "        mask = self.mask_prepare(image_id,True)\n",
    "        \n",
    "        class_ids = np.array([self.class_names.index(s) for s in chips])\n",
    "        return mask.astype(np.bool), class_ids.astype(np.int32)\n",
    "\n",
    "    def info_display(self,count):\n",
    "        #flaw_size info\n",
    "        max_flaw_size = np.max(self.all_flaw_size)\n",
    "        min_flaw_size = np.min(self.all_flaw_size)\n",
    "        mean_flaw_size = np.mean(self.all_flaw_size)\n",
    "        \n",
    "        #flaw_kind_info\n",
    "        bump_num,dent_num,dot_num = 0,0,0\n",
    "        for i in range(count):\n",
    "            chips = self.image_info[i]['chips']\n",
    "            if \"bump\" in chips:\n",
    "                bump_num +=1\n",
    "            if \"dent\" in chips:\n",
    "                dent_num +=1\n",
    "            if \"dot\" in chips:\n",
    "                dot_num += 1    \n",
    "                \n",
    "        #display all info\n",
    "        print(\"**** info of %s dataset : ***  \"%self.subset)\n",
    "        print(\"  ratio of null mask images:%d %%\"%(self.num_null_mask/count*100))\n",
    "        print(\"  max flaw size:\",max_flaw_size)\n",
    "        print(\"  min flaw size:\",min_flaw_size)\n",
    "        print(\"  mean flaw size:\",mean_flaw_size)\n",
    "        print(\"  bump ratio:%d %%\"%(bump_num/count*100))\n",
    "        print(\"  dent ratio:%d %%\"%(dent_num/count*100))\n",
    "        print(\"  dot ratio:%d %%\"%(dot_num/count*100))\n",
    "    \n",
    "dataset_test = ChipsDataset()\n",
    "dataset_test.load_chips(NUM_TEST,\"test\")\n",
    "dataset_test.prepare()\n",
    "dataset_test.info_display(NUM_TEST)\n",
    "\n",
    "# dataset_train = ChipsDataset()\n",
    "# dataset_train.load_chips(NUM_TRAIN,\"train\")\n",
    "# dataset_train.prepare()\n",
    "# dataset_train.info_display(NUM_TRAIN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading weights from  /mnt/sh_flex_storage/malu/venv/CHIPS_MRCNN/logs_chips4/chips20181016T1457/mask_rcnn_chips_0012.h5\n",
      "load weight from /mnt/sh_flex_storage/malu/venv/CHIPS_MRCNN/logs_chips4/chips20181016T1457/mask_rcnn_chips_0012.h5 by name\n",
      "Re-starting from epoch 12\n",
      "working on 0\n",
      "working on 1\n",
      "working on 2\n",
      "working on 3\n",
      "working on 4\n",
      "working on 5\n",
      "working on 6\n",
      "working on 7\n",
      "working on 8\n",
      "working on 9\n",
      "working on 10\n",
      "working on 11\n",
      "working on 12\n",
      "working on 13\n",
      "working on 14\n",
      "working on 15\n",
      "working on 16\n",
      "working on 17\n",
      "working on 18\n",
      "working on 19\n",
      "working on 20\n",
      "working on 21\n",
      "working on 22\n",
      "working on 23\n",
      "working on 24\n",
      "working on 25\n",
      "working on 26\n",
      "working on 27\n",
      "working on 28\n",
      "working on 29\n",
      "working on 30\n",
      "working on 31\n",
      "working on 32\n",
      "working on 33\n",
      "working on 34\n",
      "working on 35\n",
      "working on 36\n",
      "working on 37\n",
      "working on 38\n",
      "working on 39\n",
      "working on 40\n",
      "working on 41\n",
      "working on 42\n",
      "working on 43\n",
      "working on 44\n",
      "working on 45\n",
      "working on 46\n",
      "working on 47\n",
      "working on 48\n",
      "working on 49\n",
      "working on 50\n",
      "working on 51\n",
      "working on 52\n",
      "working on 53\n",
      "working on 54\n",
      "working on 55\n",
      "working on 56\n",
      "working on 57\n",
      "working on 58\n",
      "working on 59\n",
      "working on 60\n",
      "working on 61\n",
      "working on 62\n",
      "working on 63\n",
      "working on 64\n",
      "working on 65\n",
      "working on 66\n",
      "working on 67\n",
      "working on 68\n",
      "working on 69\n",
      "working on 70\n",
      "working on 71\n",
      "working on 72\n",
      "working on 73\n",
      "working on 74\n",
      "working on 75\n",
      "working on 76\n",
      "working on 77\n",
      "working on 78\n",
      "working on 79\n",
      "working on 80\n",
      "working on 81\n",
      "working on 82\n",
      "working on 83\n",
      "working on 84\n",
      "working on 85\n",
      "working on 86\n",
      "working on 87\n",
      "working on 88\n",
      "working on 89\n",
      "working on 90\n",
      "working on 91\n",
      "working on 92\n",
      "working on 93\n",
      "working on 94\n",
      "working on 95\n",
      "working on 96\n",
      "working on 97\n",
      "working on 98\n",
      "working on 99\n",
      "working on 100\n",
      "working on 101\n",
      "working on 102\n",
      "working on 103\n",
      "working on 104\n",
      "working on 105\n",
      "working on 106\n",
      "working on 107\n",
      "working on 108\n",
      "working on 109\n",
      "working on 110\n",
      "working on 111\n",
      "working on 112\n",
      "working on 113\n",
      "working on 114\n",
      "working on 115\n",
      "working on 116\n",
      "working on 117\n",
      "working on 118\n",
      "working on 119\n",
      "working on 120\n",
      "working on 121\n",
      "working on 122\n",
      "working on 123\n",
      "working on 124\n",
      "working on 125\n",
      "working on 126\n",
      "working on 127\n",
      "working on 128\n",
      "working on 129\n",
      "working on 130\n",
      "working on 131\n",
      "working on 132\n",
      "working on 133\n",
      "working on 134\n",
      "working on 135\n",
      "working on 136\n",
      "working on 137\n",
      "working on 138\n",
      "working on 139\n",
      "working on 140\n",
      "working on 141\n",
      "working on 142\n",
      "working on 143\n",
      "working on 144\n",
      "working on 145\n",
      "working on 146\n",
      "working on 147\n",
      "working on 148\n",
      "working on 149\n",
      "working on 150\n",
      "working on 151\n",
      "working on 152\n",
      "working on 153\n",
      "working on 154\n",
      "working on 155\n",
      "working on 156\n",
      "working on 157\n",
      "working on 158\n",
      "working on 159\n",
      "working on 160\n",
      "working on 161\n",
      "working on 162\n",
      "working on 163\n",
      "working on 164\n",
      "working on 165\n",
      "working on 166\n",
      "working on 167\n",
      "working on 168\n",
      "working on 169\n",
      "working on 170\n",
      "working on 171\n",
      "working on 172\n",
      "working on 173\n",
      "working on 174\n",
      "working on 175\n",
      "working on 176\n",
      "working on 177\n",
      "working on 178\n",
      "working on 179\n",
      "working on 180\n",
      "working on 181\n",
      "working on 182\n",
      "working on 183\n",
      "working on 184\n",
      "working on 185\n",
      "working on 186\n",
      "working on 187\n",
      "working on 188\n",
      "working on 189\n",
      "working on 190\n",
      "working on 191\n",
      "working on 192\n",
      "working on 193\n",
      "working on 194\n",
      "working on 195\n",
      "working on 196\n",
      "working on 197\n",
      "working on 198\n",
      "working on 199\n",
      "working on 200\n",
      "working on 201\n",
      "working on 202\n",
      "working on 203\n",
      "working on 204\n",
      "working on 205\n",
      "working on 206\n",
      "working on 207\n",
      "working on 208\n",
      "working on 209\n",
      "working on 210\n",
      "working on 211\n",
      "working on 212\n",
      "working on 213\n",
      "working on 214\n",
      "working on 215\n",
      "working on 216\n",
      "working on 217\n",
      "working on 218\n",
      "working on 219\n",
      "working on 220\n",
      "working on 221\n",
      "working on 222\n",
      "working on 223\n",
      "working on 224\n",
      "working on 225\n",
      "working on 226\n",
      "working on 227\n",
      "working on 228\n",
      "working on 229\n",
      "working on 230\n",
      "working on 231\n",
      "working on 232\n",
      "working on 233\n",
      "working on 234\n",
      "working on 235\n",
      "working on 236\n",
      "working on 237\n",
      "working on 238\n",
      "working on 239\n",
      "working on 240\n",
      "working on 241\n",
      "working on 242\n",
      "working on 243\n",
      "working on 244\n",
      "working on 245\n",
      "working on 246\n",
      "working on 247\n",
      "working on 248\n",
      "working on 249\n",
      "working on 250\n",
      "working on 251\n",
      "working on 252\n",
      "working on 253\n",
      "working on 254\n",
      "working on 255\n",
      "working on 256\n",
      "working on 257\n",
      "working on 258\n",
      "working on 259\n",
      "working on 260\n",
      "working on 261\n",
      "working on 262\n",
      "working on 263\n",
      "working on 264\n",
      "working on 265\n",
      "working on 266\n",
      "working on 267\n",
      "working on 268\n",
      "working on 269\n",
      "working on 270\n",
      "working on 271\n",
      "working on 272\n",
      "working on 273\n",
      "working on 274\n",
      "working on 275\n",
      "working on 276\n",
      "working on 277\n",
      "working on 278\n",
      "working on 279\n",
      "working on 280\n",
      "working on 281\n",
      "working on 282\n",
      "working on 283\n",
      "working on 284\n",
      "working on 285\n",
      "working on 286\n",
      "working on 287\n",
      "working on 288\n",
      "working on 289\n",
      "working on 290\n",
      "working on 291\n",
      "working on 292\n",
      "working on 293\n",
      "working on 294\n",
      "working on 295\n",
      "working on 296\n",
      "working on 297\n",
      "working on 298\n",
      "working on 299\n",
      "working on 300\n",
      "working on 301\n",
      "working on 302\n",
      "working on 303\n",
      "working on 304\n",
      "working on 305\n",
      "working on 306\n",
      "working on 307\n",
      "working on 308\n",
      "working on 309\n",
      "working on 310\n",
      "working on 311\n",
      "working on 312\n",
      "working on 313\n",
      "working on 314\n",
      "working on 315\n",
      "working on 316\n",
      "working on 317\n",
      "working on 318\n",
      "working on 319\n",
      "working on 320\n",
      "working on 321\n",
      "working on 322\n",
      "working on 323\n",
      "working on 324\n",
      "working on 325\n",
      "working on 326\n",
      "working on 327\n",
      "working on 328\n",
      "working on 329\n",
      "working on 330\n",
      "working on 331\n",
      "working on 332\n",
      "working on 333\n",
      "working on 334\n",
      "working on 335\n",
      "working on 336\n",
      "working on 337\n",
      "working on 338\n",
      "working on 339\n",
      "working on 340\n",
      "working on 341\n",
      "working on 342\n",
      "working on 343\n",
      "working on 344\n",
      "working on 345\n",
      "working on 346\n",
      "working on 347\n",
      "working on 348\n",
      "working on 349\n",
      "working on 350\n",
      "working on 351\n",
      "working on 352\n",
      "working on 353\n",
      "working on 354\n",
      "working on 355\n",
      "working on 356\n",
      "working on 357\n",
      "working on 358\n",
      "working on 359\n",
      "working on 360\n",
      "working on 361\n",
      "working on 362\n",
      "working on 363\n",
      "working on 364\n",
      "working on 365\n",
      "working on 366\n",
      "working on 367\n",
      "working on 368\n",
      "working on 369\n",
      "working on 370\n",
      "working on 371\n",
      "working on 372\n",
      "working on 373\n",
      "working on 374\n",
      "working on 375\n",
      "working on 376\n",
      "working on 377\n",
      "working on 378\n",
      "working on 379\n",
      "working on 380\n",
      "working on 381\n",
      "working on 382\n",
      "working on 383\n",
      "working on 384\n",
      "working on 385\n",
      "working on 386\n",
      "working on 387\n",
      "working on 388\n",
      "working on 389\n",
      "working on 390\n",
      "working on 391\n",
      "working on 392\n",
      "working on 393\n",
      "working on 394\n",
      "working on 395\n",
      "working on 396\n",
      "working on 397\n",
      "working on 398\n",
      "working on 399\n",
      "working on 400\n",
      "working on 401\n",
      "working on 402\n",
      "working on 403\n",
      "working on 404\n",
      "working on 405\n",
      "working on 406\n",
      "working on 407\n",
      "working on 408\n",
      "working on 409\n",
      "working on 410\n",
      "working on 411\n",
      "working on 412\n",
      "working on 413\n",
      "working on 414\n",
      "working on 415\n",
      "working on 416\n",
      "working on 417\n",
      "working on 418\n",
      "working on 419\n",
      "working on 420\n",
      "working on 421\n",
      "working on 422\n",
      "working on 423\n",
      "working on 424\n",
      "working on 425\n",
      "working on 426\n",
      "working on 427\n",
      "working on 428\n",
      "working on 429\n",
      "working on 430\n",
      "working on 431\n",
      "working on 432\n",
      "working on 433\n",
      "working on 434\n",
      "working on 435\n",
      "working on 436\n",
      "working on 437\n",
      "working on 438\n",
      "working on 439\n",
      "working on 440\n",
      "working on 441\n",
      "working on 442\n",
      "working on 443\n",
      "working on 444\n",
      "working on 445\n",
      "working on 446\n",
      "working on 447\n",
      "working on 448\n",
      "working on 449\n",
      "working on 450\n",
      "working on 451\n",
      "working on 452\n",
      "working on 453\n",
      "working on 454\n",
      "working on 455\n",
      "working on 456\n",
      "working on 457\n",
      "working on 458\n",
      "working on 459\n",
      "working on 460\n",
      "working on 461\n",
      "working on 462\n",
      "working on 463\n",
      "working on 464\n",
      "working on 465\n",
      "working on 466\n",
      "working on 467\n",
      "working on 468\n",
      "working on 469\n",
      "working on 470\n",
      "working on 471\n",
      "working on 472\n",
      "working on 473\n",
      "working on 474\n",
      "working on 475\n",
      "working on 476\n",
      "working on 477\n",
      "working on 478\n",
      "working on 479\n",
      "working on 480\n",
      "working on 481\n",
      "working on 482\n",
      "working on 483\n",
      "working on 484\n",
      "working on 485\n",
      "working on 486\n",
      "working on 487\n",
      "working on 488\n",
      "working on 489\n",
      "working on 490\n",
      "working on 491\n",
      "working on 492\n",
      "working on 493\n",
      "working on 494\n",
      "working on 495\n",
      "working on 496\n",
      "working on 497\n",
      "working on 498\n",
      "working on 499\n",
      "working on 500\n",
      "working on 501\n",
      "working on 502\n",
      "working on 503\n",
      "working on 504\n",
      "working on 505\n",
      "working on 506\n",
      "working on 507\n",
      "working on 508\n",
      "working on 509\n",
      "working on 510\n",
      "working on 511\n",
      "working on 512\n",
      "working on 513\n",
      "working on 514\n",
      "working on 515\n",
      "working on 516\n",
      "working on 517\n",
      "working on 518\n",
      "working on 519\n",
      "working on 520\n",
      "working on 521\n",
      "working on 522\n",
      "working on 523\n",
      "working on 524\n",
      "working on 525\n",
      "working on 526\n",
      "working on 527\n",
      "working on 528\n",
      "working on 529\n",
      "working on 530\n",
      "working on 531\n",
      "working on 532\n",
      "working on 533\n",
      "working on 534\n",
      "working on 535\n",
      "working on 536\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "working on 537\n",
      "working on 538\n",
      "working on 539\n",
      "working on 540\n",
      "working on 541\n",
      "working on 542\n",
      "working on 543\n",
      "working on 544\n",
      "working on 545\n",
      "working on 546\n",
      "working on 547\n",
      "working on 548\n",
      "working on 549\n",
      "working on 550\n",
      "working on 551\n",
      "working on 552\n",
      "working on 553\n",
      "working on 554\n",
      "working on 555\n",
      "working on 556\n",
      "working on 557\n",
      "working on 558\n",
      "working on 559\n",
      "working on 560\n",
      "working on 561\n",
      "working on 562\n",
      "working on 563\n",
      "working on 564\n",
      "working on 565\n",
      "working on 566\n",
      "working on 567\n",
      "working on 568\n",
      "working on 569\n",
      "working on 570\n",
      "working on 571\n",
      "working on 572\n",
      "working on 573\n",
      "working on 574\n",
      "working on 575\n",
      "working on 576\n",
      "working on 577\n",
      "working on 578\n",
      "working on 579\n",
      "working on 580\n",
      "working on 581\n",
      "working on 582\n",
      "working on 583\n",
      "working on 584\n",
      "working on 585\n",
      "working on 586\n",
      "working on 587\n",
      "working on 588\n",
      "working on 589\n",
      "working on 590\n",
      "working on 591\n",
      "working on 592\n",
      "working on 593\n",
      "working on 594\n",
      "working on 595\n",
      "working on 596\n",
      "working on 597\n",
      "working on 598\n",
      "working on 599\n",
      "working on 600\n",
      "working on 601\n",
      "working on 602\n",
      "working on 603\n",
      "working on 604\n",
      "working on 605\n",
      "working on 606\n",
      "working on 607\n",
      "working on 608\n",
      "working on 609\n",
      "working on 610\n",
      "working on 611\n",
      "working on 612\n",
      "working on 613\n",
      "working on 614\n",
      "working on 615\n",
      "working on 616\n",
      "working on 617\n",
      "working on 618\n",
      "working on 619\n",
      "working on 620\n",
      "working on 621\n",
      "working on 622\n",
      "working on 623\n",
      "working on 624\n",
      "working on 625\n",
      "working on 626\n",
      "working on 627\n",
      "working on 628\n",
      "working on 629\n",
      "working on 630\n",
      "working on 631\n",
      "working on 632\n",
      "working on 633\n",
      "working on 634\n",
      "working on 635\n",
      "working on 636\n",
      "working on 637\n",
      "working on 638\n",
      "working on 639\n",
      "working on 640\n",
      "working on 641\n",
      "working on 642\n",
      "working on 643\n",
      "working on 644\n",
      "working on 645\n",
      "working on 646\n",
      "working on 647\n",
      "working on 648\n",
      "working on 649\n",
      "working on 650\n",
      "working on 651\n",
      "working on 652\n",
      "working on 653\n",
      "working on 654\n",
      "working on 655\n",
      "working on 656\n",
      "working on 657\n",
      "working on 658\n",
      "working on 659\n",
      "working on 660\n",
      "working on 661\n",
      "working on 662\n",
      "working on 663\n",
      "working on 664\n",
      "working on 665\n",
      "working on 666\n",
      "working on 667\n",
      "working on 668\n",
      "working on 669\n",
      "working on 670\n",
      "working on 671\n",
      "working on 672\n",
      "working on 673\n",
      "working on 674\n",
      "working on 675\n",
      "working on 676\n",
      "working on 677\n",
      "working on 678\n",
      "working on 679\n",
      "working on 680\n",
      "working on 681\n",
      "working on 682\n",
      "working on 683\n",
      "working on 684\n",
      "working on 685\n",
      "working on 686\n",
      "working on 687\n",
      "working on 688\n",
      "working on 689\n",
      "working on 690\n",
      "working on 691\n",
      "working on 692\n",
      "working on 693\n",
      "working on 694\n",
      "working on 695\n",
      "working on 696\n",
      "working on 697\n",
      "working on 698\n",
      "working on 699\n",
      "working on 700\n",
      "working on 701\n",
      "working on 702\n",
      "working on 703\n",
      "working on 704\n",
      "working on 705\n",
      "working on 706\n",
      "working on 707\n",
      "working on 708\n",
      "working on 709\n",
      "working on 710\n",
      "working on 711\n",
      "working on 712\n",
      "working on 713\n",
      "working on 714\n",
      "working on 715\n",
      "working on 716\n",
      "working on 717\n",
      "working on 718\n",
      "working on 719\n",
      "working on 720\n",
      "working on 721\n",
      "working on 722\n",
      "working on 723\n",
      "working on 724\n",
      "working on 725\n",
      "working on 726\n",
      "working on 727\n",
      "working on 728\n",
      "working on 729\n",
      "working on 730\n",
      "working on 731\n",
      "working on 732\n",
      "working on 733\n",
      "working on 734\n",
      "working on 735\n",
      "working on 736\n",
      "working on 737\n",
      "working on 738\n",
      "working on 739\n",
      "working on 740\n",
      "working on 741\n",
      "working on 742\n",
      "working on 743\n",
      "working on 744\n",
      "working on 745\n",
      "working on 746\n",
      "working on 747\n",
      "working on 748\n",
      "working on 749\n",
      "working on 750\n",
      "working on 751\n",
      "working on 752\n",
      "working on 753\n",
      "working on 754\n",
      "working on 755\n",
      "working on 756\n",
      "working on 757\n",
      "working on 758\n",
      "working on 759\n",
      "working on 760\n",
      "working on 761\n",
      "working on 762\n",
      "working on 763\n",
      "working on 764\n",
      "working on 765\n",
      "working on 766\n",
      "working on 767\n",
      "working on 768\n",
      "working on 769\n",
      "working on 770\n",
      "working on 771\n",
      "working on 772\n",
      "working on 773\n",
      "working on 774\n",
      "working on 775\n",
      "working on 776\n",
      "working on 777\n",
      "working on 778\n",
      "working on 779\n",
      "working on 780\n",
      "working on 781\n",
      "working on 782\n",
      "working on 783\n",
      "working on 784\n",
      "working on 785\n",
      "working on 786\n",
      "working on 787\n",
      "working on 788\n",
      "working on 789\n",
      "working on 790\n",
      "working on 791\n",
      "working on 792\n",
      "working on 793\n",
      "working on 794\n",
      "working on 795\n",
      "working on 796\n",
      "working on 797\n",
      "working on 798\n",
      "working on 799\n",
      "working on 800\n",
      "working on 801\n",
      "working on 802\n",
      "working on 803\n",
      "working on 804\n",
      "working on 805\n",
      "working on 806\n",
      "working on 807\n",
      "working on 808\n",
      "working on 809\n",
      "working on 810\n",
      "working on 811\n",
      "working on 812\n",
      "working on 813\n",
      "working on 814\n",
      "working on 815\n",
      "working on 816\n",
      "working on 817\n",
      "working on 818\n",
      "working on 819\n",
      "working on 820\n",
      "working on 821\n",
      "working on 822\n",
      "working on 823\n",
      "working on 824\n",
      "working on 825\n",
      "working on 826\n",
      "working on 827\n",
      "working on 828\n",
      "working on 829\n",
      "working on 830\n",
      "working on 831\n",
      "working on 832\n",
      "working on 833\n",
      "working on 834\n",
      "working on 835\n",
      "working on 836\n",
      "working on 837\n",
      "working on 838\n",
      "working on 839\n",
      "working on 840\n",
      "working on 841\n",
      "working on 842\n",
      "working on 843\n",
      "working on 844\n",
      "working on 845\n",
      "working on 846\n",
      "working on 847\n",
      "working on 848\n",
      "working on 849\n",
      "working on 850\n",
      "working on 851\n",
      "working on 852\n",
      "working on 853\n",
      "working on 854\n",
      "working on 855\n",
      "working on 856\n",
      "working on 857\n",
      "working on 858\n",
      "working on 859\n",
      "working on 860\n",
      "working on 861\n",
      "working on 862\n",
      "working on 863\n",
      "working on 864\n",
      "working on 865\n",
      "working on 866\n",
      "working on 867\n",
      "working on 868\n",
      "working on 869\n",
      "working on 870\n",
      "working on 871\n",
      "working on 872\n",
      "working on 873\n",
      "working on 874\n",
      "working on 875\n",
      "working on 876\n",
      "working on 877\n",
      "working on 878\n",
      "working on 879\n",
      "working on 880\n",
      "working on 881\n",
      "working on 882\n",
      "working on 883\n",
      "working on 884\n",
      "working on 885\n",
      "working on 886\n",
      "working on 887\n",
      "working on 888\n",
      "working on 889\n",
      "working on 890\n",
      "working on 891\n",
      "working on 892\n",
      "working on 893\n",
      "working on 894\n",
      "working on 895\n",
      "working on 896\n",
      "working on 897\n",
      "working on 898\n",
      "working on 899\n",
      "working on 900\n",
      "working on 901\n",
      "working on 902\n",
      "working on 903\n",
      "working on 904\n",
      "working on 905\n",
      "working on 906\n",
      "working on 907\n",
      "working on 908\n",
      "working on 909\n",
      "working on 910\n",
      "working on 911\n",
      "working on 912\n",
      "working on 913\n",
      "working on 914\n",
      "working on 915\n",
      "working on 916\n",
      "working on 917\n",
      "working on 918\n",
      "working on 919\n",
      "working on 920\n",
      "working on 921\n",
      "working on 922\n",
      "working on 923\n",
      "working on 924\n",
      "working on 925\n",
      "working on 926\n",
      "working on 927\n",
      "working on 928\n",
      "working on 929\n",
      "working on 930\n",
      "working on 931\n",
      "working on 932\n",
      "working on 933\n",
      "working on 934\n",
      "working on 935\n",
      "working on 936\n",
      "working on 937\n",
      "working on 938\n",
      "working on 939\n",
      "working on 940\n",
      "working on 941\n",
      "working on 942\n",
      "working on 943\n",
      "working on 944\n",
      "working on 945\n",
      "working on 946\n",
      "working on 947\n",
      "working on 948\n",
      "working on 949\n",
      "working on 950\n",
      "working on 951\n",
      "working on 952\n",
      "working on 953\n",
      "working on 954\n",
      "working on 955\n",
      "working on 956\n",
      "working on 957\n",
      "working on 958\n",
      "working on 959\n",
      "working on 960\n",
      "working on 961\n",
      "working on 962\n",
      "working on 963\n",
      "working on 964\n",
      "working on 965\n",
      "working on 966\n",
      "working on 967\n",
      "working on 968\n",
      "working on 969\n",
      "working on 970\n",
      "working on 971\n",
      "working on 972\n",
      "working on 973\n",
      "working on 974\n",
      "working on 975\n",
      "working on 976\n",
      "working on 977\n",
      "working on 978\n",
      "working on 979\n",
      "working on 980\n",
      "working on 981\n",
      "working on 982\n",
      "working on 983\n",
      "working on 984\n",
      "working on 985\n",
      "working on 986\n",
      "working on 987\n",
      "working on 988\n",
      "working on 989\n",
      "working on 990\n",
      "working on 991\n",
      "working on 992\n",
      "working on 993\n",
      "working on 994\n",
      "working on 995\n",
      "working on 996\n",
      "working on 997\n",
      "working on 998\n",
      "working on 999\n",
      "working on 1000\n",
      "working on 1001\n",
      "working on 1002\n",
      "working on 1003\n",
      "working on 1004\n",
      "working on 1005\n",
      "working on 1006\n",
      "working on 1007\n",
      "working on 1008\n",
      "working on 1009\n",
      "working on 1010\n",
      "working on 1011\n",
      "working on 1012\n",
      "working on 1013\n",
      "working on 1014\n",
      "working on 1015\n",
      "working on 1016\n",
      "working on 1017\n",
      "working on 1018\n",
      "working on 1019\n",
      "working on 1020\n",
      "working on 1021\n",
      "working on 1022\n",
      "working on 1023\n",
      "working on 1024\n",
      "working on 1025\n",
      "working on 1026\n",
      "working on 1027\n",
      "working on 1028\n",
      "working on 1029\n",
      "working on 1030\n",
      "working on 1031\n",
      "working on 1032\n",
      "working on 1033\n",
      "working on 1034\n",
      "working on 1035\n",
      "working on 1036\n",
      "working on 1037\n",
      "working on 1038\n",
      "working on 1039\n",
      "working on 1040\n",
      "working on 1041\n",
      "working on 1042\n",
      "working on 1043\n",
      "working on 1044\n",
      "working on 1045\n",
      "working on 1046\n",
      "working on 1047\n",
      "working on 1048\n",
      "working on 1049\n",
      "working on 1050\n",
      "working on 1051\n",
      "working on 1052\n",
      "working on 1053\n",
      "working on 1054\n",
      "working on 1055\n",
      "working on 1056\n",
      "working on 1057\n",
      "working on 1058\n",
      "working on 1059\n",
      "working on 1060\n",
      "working on 1061\n",
      "working on 1062\n",
      "working on 1063\n",
      "working on 1064\n",
      "working on 1065\n",
      "working on 1066\n",
      "working on 1067\n",
      "working on 1068\n",
      "working on 1069\n",
      "working on 1070\n",
      "working on 1071\n",
      "working on 1072\n",
      "working on 1073\n",
      "working on 1074\n",
      "working on 1075\n",
      "working on 1076\n",
      "working on 1077\n",
      "working on 1078\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "working on 1079\n",
      "working on 1080\n",
      "working on 1081\n",
      "working on 1082\n",
      "working on 1083\n",
      "working on 1084\n",
      "working on 1085\n",
      "working on 1086\n",
      "working on 1087\n",
      "working on 1088\n",
      "working on 1089\n",
      "working on 1090\n",
      "working on 1091\n",
      "working on 1092\n",
      "working on 1093\n",
      "working on 1094\n",
      "working on 1095\n",
      "working on 1096\n",
      "working on 1097\n",
      "working on 1098\n",
      "working on 1099\n",
      "working on 1100\n",
      "working on 1101\n",
      "working on 1102\n",
      "working on 1103\n",
      "working on 1104\n",
      "working on 1105\n",
      "working on 1106\n",
      "working on 1107\n",
      "working on 1108\n",
      "working on 1109\n",
      "working on 1110\n",
      "working on 1111\n",
      "working on 1112\n",
      "working on 1113\n",
      "working on 1114\n",
      "working on 1115\n",
      "working on 1116\n",
      "working on 1117\n",
      "working on 1118\n",
      "working on 1119\n",
      "working on 1120\n",
      "working on 1121\n",
      "working on 1122\n",
      "working on 1123\n",
      "working on 1124\n",
      "working on 1125\n",
      "working on 1126\n",
      "working on 1127\n",
      "working on 1128\n",
      "working on 1129\n",
      "working on 1130\n",
      "working on 1131\n",
      "working on 1132\n",
      "working on 1133\n",
      "working on 1134\n",
      "working on 1135\n",
      "working on 1136\n",
      "working on 1137\n",
      "working on 1138\n",
      "working on 1139\n",
      "working on 1140\n",
      "working on 1141\n",
      "working on 1142\n",
      "working on 1143\n",
      "working on 1144\n",
      "working on 1145\n",
      "working on 1146\n",
      "working on 1147\n",
      "working on 1148\n",
      "working on 1149\n",
      "working on 1150\n",
      "working on 1151\n",
      "working on 1152\n",
      "working on 1153\n",
      "working on 1154\n",
      "working on 1155\n",
      "working on 1156\n",
      "working on 1157\n",
      "working on 1158\n",
      "working on 1159\n",
      "working on 1160\n",
      "working on 1161\n",
      "working on 1162\n",
      "working on 1163\n",
      "working on 1164\n",
      "working on 1165\n",
      "working on 1166\n",
      "working on 1167\n",
      "working on 1168\n",
      "working on 1169\n",
      "working on 1170\n",
      "working on 1171\n",
      "working on 1172\n",
      "working on 1173\n",
      "working on 1174\n",
      "working on 1175\n",
      "working on 1176\n",
      "working on 1177\n",
      "working on 1178\n",
      "working on 1179\n",
      "working on 1180\n",
      "working on 1181\n",
      "working on 1182\n",
      "working on 1183\n",
      "working on 1184\n",
      "working on 1185\n",
      "working on 1186\n",
      "working on 1187\n",
      "working on 1188\n",
      "working on 1189\n",
      "working on 1190\n",
      "working on 1191\n",
      "working on 1192\n",
      "working on 1193\n",
      "working on 1194\n",
      "working on 1195\n",
      "working on 1196\n",
      "working on 1197\n",
      "working on 1198\n",
      "working on 1199\n",
      "working on 1200\n",
      "working on 1201\n",
      "working on 1202\n",
      "working on 1203\n",
      "working on 1204\n",
      "working on 1205\n",
      "working on 1206\n",
      "working on 1207\n",
      "working on 1208\n",
      "working on 1209\n",
      "working on 1210\n",
      "working on 1211\n",
      "working on 1212\n",
      "working on 1213\n",
      "working on 1214\n",
      "working on 1215\n",
      "working on 1216\n",
      "working on 1217\n",
      "working on 1218\n",
      "working on 1219\n",
      "working on 1220\n",
      "working on 1221\n",
      "working on 1222\n",
      "working on 1223\n",
      "working on 1224\n",
      "working on 1225\n",
      "working on 1226\n",
      "working on 1227\n",
      "working on 1228\n",
      "working on 1229\n",
      "working on 1230\n",
      "working on 1231\n",
      "working on 1232\n",
      "working on 1233\n",
      "working on 1234\n",
      "working on 1235\n",
      "working on 1236\n",
      "working on 1237\n",
      "working on 1238\n",
      "working on 1239\n",
      "working on 1240\n",
      "working on 1241\n",
      "working on 1242\n",
      "working on 1243\n",
      "working on 1244\n",
      "working on 1245\n",
      "working on 1246\n",
      "working on 1247\n",
      "working on 1248\n",
      "working on 1249\n",
      "working on 1250\n",
      "working on 1251\n",
      "working on 1252\n",
      "working on 1253\n",
      "working on 1254\n",
      "working on 1255\n",
      "working on 1256\n",
      "working on 1257\n",
      "working on 1258\n",
      "working on 1259\n",
      "working on 1260\n",
      "working on 1261\n",
      "working on 1262\n",
      "working on 1263\n",
      "working on 1264\n",
      "working on 1265\n",
      "working on 1266\n",
      "working on 1267\n",
      "working on 1268\n",
      "working on 1269\n",
      "working on 1270\n",
      "working on 1271\n",
      "working on 1272\n",
      "working on 1273\n",
      "working on 1274\n",
      "working on 1275\n",
      "working on 1276\n",
      "working on 1277\n",
      "working on 1278\n",
      "working on 1279\n",
      "working on 1280\n",
      "working on 1281\n",
      "working on 1282\n",
      "working on 1283\n",
      "working on 1284\n",
      "working on 1285\n",
      "working on 1286\n",
      "working on 1287\n",
      "working on 1288\n",
      "working on 1289\n",
      "working on 1290\n",
      "working on 1291\n",
      "working on 1292\n",
      "working on 1293\n",
      "working on 1294\n",
      "working on 1295\n",
      "working on 1296\n",
      "working on 1297\n",
      "working on 1298\n",
      "working on 1299\n",
      "working on 1300\n",
      "working on 1301\n",
      "working on 1302\n",
      "working on 1303\n",
      "working on 1304\n",
      "working on 1305\n",
      "working on 1306\n",
      "working on 1307\n",
      "working on 1308\n",
      "working on 1309\n",
      "working on 1310\n",
      "working on 1311\n",
      "working on 1312\n",
      "working on 1313\n",
      "working on 1314\n",
      "working on 1315\n",
      "working on 1316\n",
      "working on 1317\n",
      "working on 1318\n",
      "working on 1319\n",
      "working on 1320\n",
      "working on 1321\n",
      "working on 1322\n",
      "working on 1323\n",
      "working on 1324\n",
      "working on 1325\n",
      "working on 1326\n",
      "working on 1327\n",
      "working on 1328\n",
      "working on 1329\n",
      "working on 1330\n",
      "working on 1331\n",
      "working on 1332\n",
      "working on 1333\n",
      "working on 1334\n",
      "working on 1335\n",
      "working on 1336\n",
      "working on 1337\n",
      "working on 1338\n",
      "working on 1339\n",
      "working on 1340\n",
      "working on 1341\n",
      "working on 1342\n",
      "working on 1343\n",
      "working on 1344\n",
      "working on 1345\n",
      "working on 1346\n",
      "working on 1347\n",
      "working on 1348\n",
      "working on 1349\n",
      "working on 1350\n",
      "working on 1351\n",
      "working on 1352\n",
      "working on 1353\n",
      "working on 1354\n",
      "working on 1355\n",
      "working on 1356\n",
      "working on 1357\n",
      "working on 1358\n",
      "working on 1359\n",
      "working on 1360\n",
      "working on 1361\n",
      "working on 1362\n",
      "working on 1363\n",
      "working on 1364\n",
      "working on 1365\n",
      "working on 1366\n",
      "working on 1367\n",
      "working on 1368\n",
      "working on 1369\n",
      "working on 1370\n",
      "working on 1371\n",
      "working on 1372\n",
      "working on 1373\n",
      "working on 1374\n",
      "working on 1375\n",
      "working on 1376\n",
      "working on 1377\n",
      "working on 1378\n",
      "working on 1379\n",
      "working on 1380\n",
      "working on 1381\n",
      "working on 1382\n",
      "working on 1383\n",
      "working on 1384\n",
      "working on 1385\n",
      "working on 1386\n",
      "working on 1387\n",
      "working on 1388\n",
      "working on 1389\n",
      "working on 1390\n",
      "working on 1391\n",
      "working on 1392\n",
      "working on 1393\n",
      "working on 1394\n",
      "working on 1395\n",
      "working on 1396\n",
      "working on 1397\n",
      "working on 1398\n",
      "working on 1399\n",
      "working on 1400\n",
      "working on 1401\n",
      "working on 1402\n",
      "working on 1403\n",
      "working on 1404\n",
      "working on 1405\n",
      "working on 1406\n",
      "working on 1407\n",
      "working on 1408\n",
      "working on 1409\n",
      "working on 1410\n",
      "working on 1411\n",
      "working on 1412\n",
      "working on 1413\n",
      "working on 1414\n",
      "working on 1415\n",
      "working on 1416\n",
      "working on 1417\n",
      "working on 1418\n",
      "working on 1419\n",
      "working on 1420\n",
      "working on 1421\n",
      "working on 1422\n",
      "working on 1423\n",
      "working on 1424\n",
      "working on 1425\n",
      "working on 1426\n",
      "working on 1427\n",
      "working on 1428\n",
      "working on 1429\n",
      "working on 1430\n",
      "working on 1431\n",
      "working on 1432\n",
      "working on 1433\n",
      "working on 1434\n",
      "working on 1435\n",
      "working on 1436\n",
      "working on 1437\n",
      "working on 1438\n",
      "working on 1439\n",
      "working on 1440\n",
      "working on 1441\n",
      "working on 1442\n",
      "working on 1443\n",
      "working on 1444\n",
      "working on 1445\n",
      "working on 1446\n",
      "working on 1447\n",
      "working on 1448\n",
      "working on 1449\n",
      "working on 1450\n",
      "working on 1451\n",
      "working on 1452\n",
      "working on 1453\n",
      "working on 1454\n",
      "working on 1455\n",
      "working on 1456\n",
      "working on 1457\n",
      "working on 1458\n",
      "working on 1459\n",
      "working on 1460\n",
      "working on 1461\n",
      "working on 1462\n",
      "working on 1463\n",
      "working on 1464\n",
      "working on 1465\n",
      "working on 1466\n",
      "working on 1467\n",
      "working on 1468\n",
      "working on 1469\n",
      "working on 1470\n",
      "working on 1471\n",
      "working on 1472\n",
      "working on 1473\n",
      "working on 1474\n",
      "working on 1475\n",
      "working on 1476\n",
      "working on 1477\n",
      "working on 1478\n",
      "working on 1479\n",
      "working on 1480\n",
      "working on 1481\n",
      "working on 1482\n",
      "working on 1483\n",
      "working on 1484\n",
      "working on 1485\n",
      "working on 1486\n",
      "working on 1487\n",
      "working on 1488\n",
      "working on 1489\n",
      "working on 1490\n",
      "working on 1491\n",
      "working on 1492\n",
      "working on 1493\n",
      "working on 1494\n",
      "working on 1495\n",
      "working on 1496\n",
      "working on 1497\n",
      "working on 1498\n",
      "working on 1499\n",
      "working on 1500\n",
      "working on 1501\n",
      "working on 1502\n",
      "working on 1503\n",
      "working on 1504\n",
      "working on 1505\n",
      "working on 1506\n",
      "working on 1507\n",
      "working on 1508\n",
      "working on 1509\n",
      "working on 1510\n",
      "working on 1511\n",
      "working on 1512\n",
      "working on 1513\n",
      "working on 1514\n",
      "working on 1515\n",
      "working on 1516\n",
      "working on 1517\n",
      "working on 1518\n",
      "working on 1519\n",
      "working on 1520\n",
      "working on 1521\n",
      "working on 1522\n",
      "working on 1523\n",
      "working on 1524\n",
      "working on 1525\n",
      "working on 1526\n",
      "working on 1527\n",
      "working on 1528\n",
      "working on 1529\n",
      "working on 1530\n",
      "working on 1531\n",
      "working on 1532\n",
      "working on 1533\n",
      "working on 1534\n",
      "working on 1535\n",
      "working on 1536\n",
      "working on 1537\n",
      "working on 1538\n",
      "working on 1539\n",
      "working on 1540\n",
      "working on 1541\n",
      "working on 1542\n",
      "working on 1543\n",
      "working on 1544\n",
      "working on 1545\n",
      "working on 1546\n",
      "working on 1547\n",
      "working on 1548\n",
      "working on 1549\n",
      "working on 1550\n",
      "working on 1551\n",
      "working on 1552\n",
      "working on 1553\n",
      "working on 1554\n",
      "working on 1555\n",
      "working on 1556\n",
      "working on 1557\n",
      "working on 1558\n",
      "working on 1559\n",
      "working on 1560\n",
      "working on 1561\n",
      "working on 1562\n",
      "working on 1563\n",
      "working on 1564\n",
      "working on 1565\n",
      "working on 1566\n",
      "working on 1567\n",
      "working on 1568\n",
      "working on 1569\n",
      "working on 1570\n",
      "working on 1571\n",
      "working on 1572\n",
      "working on 1573\n",
      "working on 1574\n",
      "working on 1575\n",
      "working on 1576\n",
      "working on 1577\n",
      "working on 1578\n",
      "working on 1579\n",
      "working on 1580\n",
      "working on 1581\n",
      "working on 1582\n",
      "working on 1583\n",
      "working on 1584\n",
      "working on 1585\n",
      "working on 1586\n",
      "working on 1587\n",
      "working on 1588\n",
      "working on 1589\n",
      "working on 1590\n",
      "working on 1591\n",
      "working on 1592\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "working on 1593\n",
      "working on 1594\n",
      "working on 1595\n",
      "working on 1596\n",
      "working on 1597\n",
      "working on 1598\n",
      "working on 1599\n",
      "working on 1600\n",
      "working on 1601\n",
      "working on 1602\n",
      "working on 1603\n",
      "working on 1604\n",
      "working on 1605\n",
      "working on 1606\n",
      "working on 1607\n",
      "working on 1608\n",
      "working on 1609\n",
      "working on 1610\n",
      "working on 1611\n",
      "working on 1612\n",
      "working on 1613\n",
      "working on 1614\n",
      "working on 1615\n",
      "working on 1616\n",
      "working on 1617\n",
      "working on 1618\n",
      "working on 1619\n",
      "working on 1620\n",
      "working on 1621\n",
      "working on 1622\n",
      "working on 1623\n",
      "working on 1624\n",
      "working on 1625\n",
      "working on 1626\n",
      "working on 1627\n",
      "working on 1628\n",
      "working on 1629\n",
      "working on 1630\n",
      "working on 1631\n",
      "working on 1632\n",
      "working on 1633\n",
      "working on 1634\n",
      "working on 1635\n",
      "working on 1636\n",
      "working on 1637\n",
      "working on 1638\n",
      "working on 1639\n",
      "working on 1640\n",
      "working on 1641\n",
      "working on 1642\n",
      "working on 1643\n",
      "working on 1644\n",
      "working on 1645\n",
      "working on 1646\n",
      "working on 1647\n",
      "working on 1648\n",
      "working on 1649\n",
      "working on 1650\n",
      "working on 1651\n",
      "working on 1652\n",
      "working on 1653\n",
      "working on 1654\n",
      "working on 1655\n",
      "working on 1656\n",
      "working on 1657\n",
      "working on 1658\n",
      "working on 1659\n",
      "working on 1660\n",
      "working on 1661\n",
      "working on 1662\n",
      "working on 1663\n",
      "working on 1664\n",
      "working on 1665\n",
      "working on 1666\n",
      "working on 1667\n",
      "working on 1668\n",
      "working on 1669\n",
      "working on 1670\n",
      "working on 1671\n",
      "working on 1672\n",
      "working on 1673\n",
      "working on 1674\n",
      "working on 1675\n",
      "working on 1676\n",
      "working on 1677\n",
      "working on 1678\n",
      "working on 1679\n",
      "working on 1680\n",
      "working on 1681\n",
      "working on 1682\n",
      "working on 1683\n",
      "working on 1684\n",
      "working on 1685\n",
      "working on 1686\n",
      "working on 1687\n",
      "working on 1688\n",
      "working on 1689\n",
      "working on 1690\n",
      "working on 1691\n",
      "working on 1692\n",
      "working on 1693\n",
      "working on 1694\n",
      "working on 1695\n",
      "working on 1696\n",
      "working on 1697\n",
      "working on 1698\n",
      "working on 1699\n",
      "working on 1700\n",
      "working on 1701\n",
      "working on 1702\n",
      "working on 1703\n",
      "working on 1704\n",
      "working on 1705\n",
      "working on 1706\n",
      "working on 1707\n",
      "working on 1708\n",
      "working on 1709\n",
      "working on 1710\n",
      "working on 1711\n",
      "working on 1712\n",
      "working on 1713\n",
      "working on 1714\n",
      "working on 1715\n",
      "working on 1716\n",
      "working on 1717\n",
      "working on 1718\n",
      "working on 1719\n",
      "working on 1720\n",
      "working on 1721\n",
      "working on 1722\n",
      "working on 1723\n",
      "working on 1724\n",
      "working on 1725\n",
      "working on 1726\n",
      "working on 1727\n",
      "working on 1728\n",
      "working on 1729\n",
      "working on 1730\n",
      "working on 1731\n",
      "working on 1732\n",
      "working on 1733\n",
      "working on 1734\n",
      "working on 1735\n",
      "working on 1736\n",
      "working on 1737\n",
      "working on 1738\n",
      "working on 1739\n",
      "working on 1740\n",
      "working on 1741\n",
      "working on 1742\n",
      "working on 1743\n",
      "working on 1744\n",
      "working on 1745\n",
      "working on 1746\n",
      "working on 1747\n",
      "working on 1748\n",
      "working on 1749\n",
      "working on 1750\n",
      "working on 1751\n",
      "working on 1752\n",
      "working on 1753\n",
      "working on 1754\n",
      "working on 1755\n",
      "working on 1756\n",
      "working on 1757\n",
      "working on 1758\n",
      "working on 1759\n",
      "working on 1760\n",
      "working on 1761\n",
      "working on 1762\n",
      "working on 1763\n",
      "working on 1764\n",
      "working on 1765\n",
      "working on 1766\n",
      "working on 1767\n",
      "working on 1768\n",
      "working on 1769\n",
      "working on 1770\n",
      "working on 1771\n",
      "working on 1772\n",
      "working on 1773\n",
      "working on 1774\n",
      "working on 1775\n",
      "working on 1776\n",
      "working on 1777\n",
      "working on 1778\n",
      "working on 1779\n",
      "working on 1780\n",
      "working on 1781\n",
      "working on 1782\n",
      "working on 1783\n",
      "working on 1784\n",
      "working on 1785\n",
      "working on 1786\n",
      "working on 1787\n",
      "working on 1788\n",
      "working on 1789\n",
      "working on 1790\n",
      "working on 1791\n",
      "working on 1792\n",
      "working on 1793\n",
      "working on 1794\n",
      "working on 1795\n",
      "working on 1796\n",
      "working on 1797\n",
      "working on 1798\n",
      "working on 1799\n",
      "working on 1800\n",
      "working on 1801\n",
      "working on 1802\n",
      "working on 1803\n",
      "working on 1804\n",
      "working on 1805\n",
      "working on 1806\n",
      "working on 1807\n",
      "working on 1808\n",
      "working on 1809\n",
      "working on 1810\n",
      "working on 1811\n",
      "working on 1812\n",
      "working on 1813\n",
      "working on 1814\n",
      "working on 1815\n",
      "working on 1816\n",
      "working on 1817\n",
      "working on 1818\n",
      "working on 1819\n",
      "working on 1820\n",
      "working on 1821\n",
      "working on 1822\n",
      "working on 1823\n",
      "working on 1824\n",
      "working on 1825\n",
      "working on 1826\n",
      "working on 1827\n",
      "working on 1828\n",
      "working on 1829\n",
      "working on 1830\n",
      "working on 1831\n",
      "working on 1832\n",
      "working on 1833\n",
      "working on 1834\n",
      "working on 1835\n",
      "working on 1836\n",
      "working on 1837\n",
      "working on 1838\n",
      "working on 1839\n",
      "working on 1840\n",
      "working on 1841\n",
      "working on 1842\n",
      "working on 1843\n",
      "working on 1844\n",
      "working on 1845\n",
      "working on 1846\n",
      "working on 1847\n",
      "working on 1848\n",
      "working on 1849\n",
      "working on 1850\n",
      "working on 1851\n",
      "working on 1852\n",
      "working on 1853\n",
      "working on 1854\n",
      "working on 1855\n",
      "working on 1856\n",
      "working on 1857\n",
      "working on 1858\n",
      "working on 1859\n",
      "working on 1860\n",
      "working on 1861\n",
      "working on 1862\n",
      "working on 1863\n",
      "working on 1864\n",
      "working on 1865\n",
      "working on 1866\n",
      "working on 1867\n",
      "working on 1868\n",
      "working on 1869\n",
      "working on 1870\n",
      "working on 1871\n",
      "working on 1872\n",
      "working on 1873\n",
      "working on 1874\n",
      "working on 1875\n",
      "working on 1876\n",
      "working on 1877\n",
      "working on 1878\n",
      "working on 1879\n",
      "working on 1880\n",
      "working on 1881\n",
      "working on 1882\n",
      "working on 1883\n",
      "working on 1884\n",
      "working on 1885\n",
      "working on 1886\n",
      "working on 1887\n",
      "working on 1888\n",
      "working on 1889\n",
      "working on 1890\n",
      "working on 1891\n",
      "working on 1892\n",
      "working on 1893\n",
      "working on 1894\n",
      "working on 1895\n",
      "working on 1896\n",
      "working on 1897\n",
      "working on 1898\n",
      "working on 1899\n",
      "working on 1900\n",
      "working on 1901\n",
      "working on 1902\n",
      "working on 1903\n",
      "working on 1904\n",
      "working on 1905\n",
      "working on 1906\n",
      "working on 1907\n",
      "working on 1908\n",
      "working on 1909\n",
      "working on 1910\n",
      "working on 1911\n",
      "working on 1912\n",
      "working on 1913\n",
      "working on 1914\n",
      "working on 1915\n",
      "working on 1916\n",
      "working on 1917\n",
      "working on 1918\n",
      "working on 1919\n",
      "working on 1920\n",
      "working on 1921\n",
      "working on 1922\n",
      "working on 1923\n",
      "working on 1924\n",
      "working on 1925\n",
      "working on 1926\n",
      "working on 1927\n",
      "working on 1928\n",
      "working on 1929\n",
      "working on 1930\n",
      "working on 1931\n",
      "working on 1932\n",
      "working on 1933\n",
      "working on 1934\n",
      "working on 1935\n",
      "working on 1936\n",
      "working on 1937\n",
      "working on 1938\n",
      "working on 1939\n",
      "working on 1940\n",
      "working on 1941\n",
      "working on 1942\n",
      "working on 1943\n",
      "working on 1944\n",
      "working on 1945\n",
      "working on 1946\n",
      "working on 1947\n",
      "working on 1948\n",
      "working on 1949\n",
      "working on 1950\n",
      "working on 1951\n",
      "working on 1952\n",
      "working on 1953\n",
      "working on 1954\n",
      "working on 1955\n",
      "working on 1956\n",
      "working on 1957\n",
      "working on 1958\n",
      "working on 1959\n",
      "working on 1960\n",
      "working on 1961\n",
      "working on 1962\n",
      "working on 1963\n",
      "working on 1964\n",
      "working on 1965\n",
      "working on 1966\n",
      "working on 1967\n",
      "working on 1968\n",
      "working on 1969\n",
      "working on 1970\n",
      "working on 1971\n",
      "working on 1972\n",
      "working on 1973\n",
      "working on 1974\n",
      "working on 1975\n",
      "working on 1976\n",
      "working on 1977\n",
      "working on 1978\n",
      "working on 1979\n",
      "working on 1980\n",
      "working on 1981\n",
      "working on 1982\n",
      "working on 1983\n",
      "working on 1984\n",
      "working on 1985\n",
      "working on 1986\n",
      "working on 1987\n",
      "working on 1988\n",
      "working on 1989\n",
      "working on 1990\n",
      "working on 1991\n",
      "working on 1992\n",
      "working on 1993\n",
      "working on 1994\n",
      "working on 1995\n",
      "working on 1996\n",
      "working on 1997\n",
      "working on 1998\n",
      "working on 1999\n",
      "working on 2000\n",
      "working on 2001\n",
      "working on 2002\n",
      "working on 2003\n",
      "working on 2004\n",
      "working on 2005\n",
      "working on 2006\n",
      "working on 2007\n",
      "working on 2008\n",
      "working on 2009\n",
      "working on 2010\n",
      "working on 2011\n",
      "working on 2012\n",
      "working on 2013\n",
      "working on 2014\n",
      "working on 2015\n",
      "working on 2016\n",
      "working on 2017\n",
      "working on 2018\n",
      "working on 2019\n",
      "working on 2020\n",
      "working on 2021\n",
      "working on 2022\n",
      "working on 2023\n",
      "working on 2024\n",
      "working on 2025\n",
      "working on 2026\n",
      "working on 2027\n",
      "working on 2028\n",
      "working on 2029\n",
      "working on 2030\n",
      "working on 2031\n",
      "working on 2032\n",
      "working on 2033\n",
      "working on 2034\n",
      "working on 2035\n",
      "working on 2036\n",
      "working on 2037\n",
      "working on 2038\n",
      "working on 2039\n",
      "working on 2040\n",
      "working on 2041\n",
      "working on 2042\n",
      "working on 2043\n",
      "working on 2044\n",
      "working on 2045\n",
      "working on 2046\n",
      "working on 2047\n",
      "working on 2048\n",
      "working on 2049\n",
      "working on 2050\n",
      "working on 2051\n",
      "working on 2052\n",
      "working on 2053\n",
      "working on 2054\n",
      "working on 2055\n",
      "working on 2056\n",
      "working on 2057\n",
      "working on 2058\n",
      "working on 2059\n",
      "working on 2060\n",
      "working on 2061\n",
      "working on 2062\n",
      "working on 2063\n",
      "working on 2064\n",
      "working on 2065\n",
      "working on 2066\n",
      "working on 2067\n",
      "working on 2068\n",
      "working on 2069\n",
      "working on 2070\n",
      "working on 2071\n",
      "working on 2072\n",
      "working on 2073\n",
      "working on 2074\n",
      "working on 2075\n",
      "working on 2076\n",
      "working on 2077\n",
      "working on 2078\n",
      "working on 2079\n",
      "working on 2080\n",
      "working on 2081\n",
      "working on 2082\n",
      "working on 2083\n",
      "working on 2084\n",
      "working on 2085\n",
      "working on 2086\n",
      "working on 2087\n",
      "working on 2088\n",
      "working on 2089\n",
      "working on 2090\n",
      "working on 2091\n",
      "working on 2092\n",
      "working on 2093\n",
      "working on 2094\n",
      "working on 2095\n",
      "working on 2096\n",
      "working on 2097\n",
      "working on 2098\n",
      "working on 2099\n",
      "working on 2100\n",
      "working on 2101\n",
      "working on 2102\n",
      "working on 2103\n",
      "working on 2104\n",
      "working on 2105\n",
      "working on 2106\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "working on 2107\n",
      "working on 2108\n",
      "working on 2109\n",
      "working on 2110\n",
      "working on 2111\n",
      "working on 2112\n",
      "working on 2113\n",
      "working on 2114\n",
      "working on 2115\n",
      "working on 2116\n",
      "working on 2117\n",
      "working on 2118\n",
      "working on 2119\n",
      "working on 2120\n",
      "working on 2121\n",
      "working on 2122\n",
      "working on 2123\n",
      "working on 2124\n",
      "working on 2125\n",
      "working on 2126\n",
      "working on 2127\n",
      "working on 2128\n",
      "working on 2129\n",
      "working on 2130\n",
      "working on 2131\n",
      "working on 2132\n",
      "working on 2133\n",
      "working on 2134\n",
      "working on 2135\n",
      "working on 2136\n",
      "working on 2137\n",
      "working on 2138\n",
      "working on 2139\n",
      "working on 2140\n",
      "working on 2141\n",
      "working on 2142\n",
      "working on 2143\n",
      "working on 2144\n",
      "working on 2145\n",
      "working on 2146\n",
      "working on 2147\n",
      "working on 2148\n",
      "working on 2149\n",
      "working on 2150\n",
      "working on 2151\n",
      "working on 2152\n",
      "working on 2153\n",
      "working on 2154\n",
      "working on 2155\n",
      "working on 2156\n",
      "working on 2157\n",
      "working on 2158\n",
      "working on 2159\n",
      "working on 2160\n",
      "working on 2161\n",
      "working on 2162\n",
      "working on 2163\n",
      "working on 2164\n",
      "working on 2165\n",
      "working on 2166\n",
      "working on 2167\n",
      "working on 2168\n",
      "working on 2169\n",
      "working on 2170\n",
      "working on 2171\n",
      "working on 2172\n",
      "working on 2173\n",
      "working on 2174\n",
      "working on 2175\n",
      "working on 2176\n",
      "working on 2177\n",
      "working on 2178\n",
      "working on 2179\n",
      "working on 2180\n",
      "working on 2181\n",
      "working on 2182\n",
      "working on 2183\n",
      "working on 2184\n",
      "working on 2185\n",
      "working on 2186\n",
      "working on 2187\n",
      "working on 2188\n",
      "working on 2189\n",
      "working on 2190\n",
      "working on 2191\n",
      "working on 2192\n",
      "working on 2193\n",
      "working on 2194\n",
      "working on 2195\n",
      "working on 2196\n",
      "working on 2197\n",
      "working on 2198\n",
      "working on 2199\n",
      "working on 2200\n",
      "working on 2201\n",
      "working on 2202\n",
      "working on 2203\n",
      "working on 2204\n",
      "working on 2205\n",
      "working on 2206\n",
      "working on 2207\n",
      "working on 2208\n",
      "working on 2209\n",
      "working on 2210\n",
      "working on 2211\n",
      "working on 2212\n",
      "working on 2213\n",
      "working on 2214\n",
      "working on 2215\n",
      "working on 2216\n",
      "working on 2217\n",
      "working on 2218\n",
      "working on 2219\n",
      "working on 2220\n",
      "working on 2221\n",
      "working on 2222\n",
      "working on 2223\n",
      "working on 2224\n",
      "working on 2225\n",
      "working on 2226\n",
      "working on 2227\n",
      "working on 2228\n",
      "working on 2229\n",
      "working on 2230\n",
      "working on 2231\n",
      "working on 2232\n",
      "working on 2233\n",
      "working on 2234\n",
      "working on 2235\n",
      "working on 2236\n",
      "working on 2237\n",
      "working on 2238\n",
      "working on 2239\n",
      "working on 2240\n",
      "working on 2241\n",
      "working on 2242\n",
      "working on 2243\n",
      "working on 2244\n",
      "working on 2245\n",
      "working on 2246\n",
      "working on 2247\n",
      "working on 2248\n",
      "working on 2249\n",
      "working on 2250\n",
      "working on 2251\n",
      "working on 2252\n",
      "working on 2253\n",
      "working on 2254\n",
      "working on 2255\n",
      "working on 2256\n",
      "working on 2257\n",
      "working on 2258\n",
      "working on 2259\n",
      "working on 2260\n",
      "working on 2261\n",
      "working on 2262\n",
      "working on 2263\n",
      "working on 2264\n",
      "working on 2265\n",
      "working on 2266\n",
      "working on 2267\n",
      "working on 2268\n",
      "working on 2269\n",
      "working on 2270\n",
      "working on 2271\n",
      "working on 2272\n",
      "working on 2273\n",
      "working on 2274\n",
      "working on 2275\n",
      "working on 2276\n",
      "working on 2277\n",
      "working on 2278\n",
      "working on 2279\n",
      "working on 2280\n",
      "working on 2281\n",
      "working on 2282\n",
      "working on 2283\n",
      "working on 2284\n",
      "working on 2285\n",
      "working on 2286\n",
      "working on 2287\n",
      "working on 2288\n",
      "working on 2289\n",
      "working on 2290\n",
      "working on 2291\n",
      "working on 2292\n",
      "working on 2293\n",
      "working on 2294\n",
      "working on 2295\n",
      "working on 2296\n",
      "working on 2297\n",
      "working on 2298\n",
      "working on 2299\n",
      "working on 2300\n",
      "working on 2301\n",
      "working on 2302\n",
      "working on 2303\n",
      "working on 2304\n",
      "working on 2305\n",
      "working on 2306\n",
      "working on 2307\n",
      "working on 2308\n",
      "working on 2309\n",
      "working on 2310\n",
      "working on 2311\n",
      "working on 2312\n",
      "working on 2313\n",
      "working on 2314\n",
      "working on 2315\n",
      "working on 2316\n",
      "working on 2317\n",
      "working on 2318\n",
      "working on 2319\n",
      "working on 2320\n",
      "working on 2321\n",
      "working on 2322\n",
      "working on 2323\n",
      "working on 2324\n",
      "working on 2325\n",
      "working on 2326\n",
      "working on 2327\n",
      "working on 2328\n",
      "working on 2329\n",
      "working on 2330\n",
      "working on 2331\n",
      "working on 2332\n",
      "working on 2333\n",
      "working on 2334\n",
      "working on 2335\n",
      "working on 2336\n",
      "working on 2337\n",
      "working on 2338\n",
      "working on 2339\n",
      "working on 2340\n",
      "working on 2341\n",
      "working on 2342\n",
      "working on 2343\n",
      "working on 2344\n",
      "working on 2345\n",
      "working on 2346\n",
      "working on 2347\n",
      "working on 2348\n",
      "working on 2349\n",
      "working on 2350\n",
      "working on 2351\n",
      "working on 2352\n",
      "working on 2353\n",
      "working on 2354\n",
      "working on 2355\n",
      "working on 2356\n",
      "working on 2357\n",
      "working on 2358\n",
      "working on 2359\n",
      "working on 2360\n",
      "working on 2361\n",
      "working on 2362\n",
      "working on 2363\n",
      "working on 2364\n",
      "working on 2365\n",
      "working on 2366\n",
      "working on 2367\n",
      "working on 2368\n",
      "working on 2369\n",
      "working on 2370\n",
      "working on 2371\n",
      "working on 2372\n",
      "working on 2373\n",
      "working on 2374\n",
      "working on 2375\n",
      "working on 2376\n",
      "working on 2377\n",
      "working on 2378\n",
      "working on 2379\n",
      "working on 2380\n",
      "working on 2381\n",
      "working on 2382\n",
      "working on 2383\n",
      "working on 2384\n",
      "working on 2385\n",
      "working on 2386\n",
      "working on 2387\n",
      "working on 2388\n",
      "working on 2389\n",
      "working on 2390\n",
      "working on 2391\n",
      "working on 2392\n",
      "working on 2393\n",
      "working on 2394\n",
      "working on 2395\n",
      "working on 2396\n",
      "working on 2397\n",
      "working on 2398\n",
      "working on 2399\n",
      "working on 2400\n",
      "working on 2401\n",
      "working on 2402\n",
      "working on 2403\n",
      "working on 2404\n",
      "working on 2405\n",
      "working on 2406\n",
      "working on 2407\n",
      "working on 2408\n",
      "working on 2409\n",
      "working on 2410\n",
      "working on 2411\n",
      "working on 2412\n",
      "working on 2413\n",
      "working on 2414\n",
      "working on 2415\n",
      "working on 2416\n",
      "working on 2417\n",
      "working on 2418\n",
      "working on 2419\n",
      "working on 2420\n",
      "working on 2421\n",
      "working on 2422\n",
      "working on 2423\n",
      "working on 2424\n",
      "working on 2425\n",
      "working on 2426\n",
      "working on 2427\n",
      "working on 2428\n",
      "working on 2429\n",
      "working on 2430\n",
      "working on 2431\n",
      "working on 2432\n",
      "working on 2433\n",
      "working on 2434\n",
      "working on 2435\n",
      "working on 2436\n",
      "working on 2437\n",
      "working on 2438\n",
      "working on 2439\n",
      "working on 2440\n",
      "working on 2441\n",
      "working on 2442\n",
      "working on 2443\n",
      "working on 2444\n",
      "working on 2445\n",
      "working on 2446\n",
      "working on 2447\n",
      "working on 2448\n",
      "working on 2449\n",
      "working on 2450\n",
      "working on 2451\n",
      "working on 2452\n",
      "working on 2453\n",
      "working on 2454\n",
      "working on 2455\n",
      "working on 2456\n",
      "working on 2457\n",
      "working on 2458\n",
      "working on 2459\n",
      "working on 2460\n",
      "working on 2461\n",
      "working on 2462\n",
      "working on 2463\n",
      "working on 2464\n",
      "working on 2465\n",
      "working on 2466\n",
      "working on 2467\n",
      "working on 2468\n",
      "working on 2469\n",
      "working on 2470\n",
      "working on 2471\n",
      "working on 2472\n",
      "working on 2473\n",
      "working on 2474\n",
      "working on 2475\n",
      "working on 2476\n",
      "working on 2477\n",
      "working on 2478\n",
      "working on 2479\n",
      "working on 2480\n",
      "working on 2481\n",
      "working on 2482\n",
      "working on 2483\n",
      "working on 2484\n",
      "working on 2485\n",
      "working on 2486\n",
      "working on 2487\n",
      "working on 2488\n",
      "working on 2489\n",
      "working on 2490\n",
      "working on 2491\n",
      "working on 2492\n",
      "working on 2493\n",
      "working on 2494\n",
      "working on 2495\n",
      "working on 2496\n",
      "working on 2497\n",
      "working on 2498\n",
      "working on 2499\n",
      "working on 2500\n",
      "working on 2501\n",
      "working on 2502\n",
      "working on 2503\n",
      "working on 2504\n",
      "working on 2505\n",
      "working on 2506\n",
      "working on 2507\n",
      "working on 2508\n",
      "working on 2509\n",
      "working on 2510\n",
      "working on 2511\n",
      "working on 2512\n",
      "working on 2513\n",
      "working on 2514\n",
      "working on 2515\n",
      "working on 2516\n",
      "working on 2517\n",
      "working on 2518\n",
      "working on 2519\n",
      "working on 2520\n",
      "working on 2521\n",
      "working on 2522\n",
      "working on 2523\n",
      "working on 2524\n",
      "working on 2525\n",
      "working on 2526\n",
      "working on 2527\n",
      "working on 2528\n",
      "working on 2529\n",
      "working on 2530\n",
      "working on 2531\n",
      "working on 2532\n",
      "working on 2533\n",
      "working on 2534\n",
      "working on 2535\n",
      "working on 2536\n",
      "working on 2537\n",
      "working on 2538\n",
      "working on 2539\n",
      "working on 2540\n",
      "working on 2541\n",
      "working on 2542\n",
      "working on 2543\n",
      "working on 2544\n",
      "working on 2545\n",
      "working on 2546\n",
      "working on 2547\n",
      "working on 2548\n",
      "working on 2549\n",
      "working on 2550\n",
      "working on 2551\n",
      "working on 2552\n",
      "working on 2553\n",
      "working on 2554\n",
      "working on 2555\n",
      "working on 2556\n",
      "working on 2557\n",
      "working on 2558\n",
      "working on 2559\n",
      "working on 2560\n",
      "working on 2561\n",
      "working on 2562\n",
      "working on 2563\n",
      "working on 2564\n",
      "working on 2565\n",
      "working on 2566\n",
      "working on 2567\n",
      "working on 2568\n",
      "working on 2569\n",
      "working on 2570\n",
      "working on 2571\n",
      "working on 2572\n",
      "working on 2573\n",
      "working on 2574\n",
      "working on 2575\n",
      "working on 2576\n",
      "working on 2577\n",
      "working on 2578\n",
      "working on 2579\n",
      "working on 2580\n",
      "working on 2581\n",
      "working on 2582\n",
      "working on 2583\n",
      "working on 2584\n",
      "working on 2585\n",
      "working on 2586\n",
      "working on 2587\n",
      "working on 2588\n",
      "working on 2589\n",
      "working on 2590\n",
      "working on 2591\n",
      "working on 2592\n",
      "working on 2593\n",
      "working on 2594\n",
      "working on 2595\n",
      "working on 2596\n",
      "working on 2597\n",
      "working on 2598\n",
      "working on 2599\n",
      "working on 2600\n",
      "working on 2601\n",
      "working on 2602\n",
      "working on 2603\n",
      "working on 2604\n",
      "working on 2605\n",
      "working on 2606\n",
      "working on 2607\n",
      "working on 2608\n",
      "working on 2609\n",
      "working on 2610\n",
      "working on 2611\n",
      "working on 2612\n",
      "working on 2613\n",
      "working on 2614\n",
      "working on 2615\n",
      "working on 2616\n",
      "working on 2617\n",
      "working on 2618\n",
      "working on 2619\n",
      "working on 2620\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "working on 2621\n",
      "working on 2622\n",
      "working on 2623\n",
      "working on 2624\n",
      "working on 2625\n",
      "working on 2626\n",
      "working on 2627\n",
      "working on 2628\n",
      "working on 2629\n",
      "working on 2630\n",
      "working on 2631\n",
      "working on 2632\n",
      "working on 2633\n",
      "working on 2634\n",
      "working on 2635\n",
      "working on 2636\n",
      "working on 2637\n",
      "working on 2638\n",
      "working on 2639\n",
      "working on 2640\n",
      "working on 2641\n",
      "working on 2642\n",
      "working on 2643\n",
      "working on 2644\n",
      "working on 2645\n",
      "working on 2646\n",
      "working on 2647\n",
      "working on 2648\n",
      "working on 2649\n",
      "working on 2650\n",
      "working on 2651\n",
      "working on 2652\n",
      "working on 2653\n",
      "working on 2654\n",
      "working on 2655\n",
      "working on 2656\n",
      "working on 2657\n",
      "working on 2658\n",
      "working on 2659\n",
      "working on 2660\n",
      "working on 2661\n",
      "working on 2662\n",
      "working on 2663\n",
      "working on 2664\n",
      "working on 2665\n",
      "working on 2666\n",
      "working on 2667\n",
      "working on 2668\n",
      "working on 2669\n",
      "working on 2670\n",
      "working on 2671\n",
      "working on 2672\n",
      "working on 2673\n",
      "working on 2674\n",
      "working on 2675\n",
      "working on 2676\n",
      "working on 2677\n",
      "working on 2678\n",
      "working on 2679\n",
      "working on 2680\n",
      "working on 2681\n",
      "working on 2682\n",
      "working on 2683\n",
      "working on 2684\n",
      "working on 2685\n",
      "working on 2686\n",
      "working on 2687\n",
      "working on 2688\n",
      "working on 2689\n",
      "working on 2690\n",
      "working on 2691\n",
      "working on 2692\n",
      "working on 2693\n",
      "working on 2694\n",
      "working on 2695\n",
      "working on 2696\n",
      "working on 2697\n",
      "working on 2698\n",
      "working on 2699\n",
      "working on 2700\n",
      "working on 2701\n",
      "working on 2702\n",
      "working on 2703\n",
      "working on 2704\n",
      "working on 2705\n",
      "working on 2706\n",
      "working on 2707\n",
      "working on 2708\n",
      "working on 2709\n",
      "working on 2710\n",
      "working on 2711\n",
      "working on 2712\n",
      "working on 2713\n",
      "working on 2714\n",
      "working on 2715\n",
      "working on 2716\n",
      "working on 2717\n",
      "working on 2718\n",
      "working on 2719\n",
      "working on 2720\n",
      "working on 2721\n",
      "working on 2722\n",
      "working on 2723\n",
      "working on 2724\n",
      "working on 2725\n",
      "working on 2726\n",
      "working on 2727\n",
      "working on 2728\n",
      "working on 2729\n",
      "working on 2730\n",
      "working on 2731\n",
      "working on 2732\n",
      "working on 2733\n",
      "working on 2734\n",
      "working on 2735\n",
      "working on 2736\n",
      "working on 2737\n",
      "working on 2738\n",
      "working on 2739\n",
      "working on 2740\n",
      "working on 2741\n",
      "working on 2742\n",
      "working on 2743\n",
      "working on 2744\n",
      "working on 2745\n",
      "working on 2746\n",
      "working on 2747\n",
      "working on 2748\n",
      "working on 2749\n",
      "working on 2750\n",
      "working on 2751\n",
      "working on 2752\n",
      "working on 2753\n",
      "working on 2754\n",
      "working on 2755\n",
      "working on 2756\n",
      "working on 2757\n",
      "working on 2758\n",
      "working on 2759\n",
      "working on 2760\n",
      "working on 2761\n",
      "working on 2762\n",
      "working on 2763\n",
      "working on 2764\n",
      "working on 2765\n",
      "working on 2766\n",
      "working on 2767\n",
      "working on 2768\n",
      "working on 2769\n",
      "working on 2770\n",
      "working on 2771\n",
      "working on 2772\n",
      "working on 2773\n",
      "working on 2774\n",
      "working on 2775\n",
      "working on 2776\n",
      "working on 2777\n",
      "working on 2778\n",
      "working on 2779\n",
      "working on 2780\n",
      "working on 2781\n",
      "working on 2782\n",
      "working on 2783\n",
      "working on 2784\n",
      "working on 2785\n",
      "working on 2786\n",
      "working on 2787\n",
      "working on 2788\n",
      "working on 2789\n",
      "working on 2790\n",
      "working on 2791\n",
      "working on 2792\n",
      "working on 2793\n",
      "working on 2794\n",
      "working on 2795\n",
      "working on 2796\n",
      "working on 2797\n",
      "working on 2798\n",
      "working on 2799\n",
      "working on 2800\n",
      "working on 2801\n",
      "working on 2802\n",
      "working on 2803\n",
      "working on 2804\n",
      "working on 2805\n",
      "working on 2806\n",
      "working on 2807\n",
      "working on 2808\n",
      "working on 2809\n",
      "working on 2810\n",
      "working on 2811\n",
      "working on 2812\n",
      "working on 2813\n",
      "working on 2814\n",
      "working on 2815\n",
      "working on 2816\n",
      "working on 2817\n",
      "working on 2818\n",
      "working on 2819\n",
      "working on 2820\n",
      "working on 2821\n",
      "working on 2822\n",
      "working on 2823\n",
      "working on 2824\n",
      "working on 2825\n",
      "working on 2826\n",
      "working on 2827\n",
      "working on 2828\n",
      "working on 2829\n",
      "working on 2830\n",
      "working on 2831\n",
      "working on 2832\n",
      "working on 2833\n",
      "working on 2834\n",
      "working on 2835\n",
      "working on 2836\n",
      "working on 2837\n",
      "working on 2838\n",
      "working on 2839\n",
      "working on 2840\n",
      "working on 2841\n",
      "working on 2842\n",
      "working on 2843\n",
      "working on 2844\n",
      "working on 2845\n",
      "working on 2846\n",
      "working on 2847\n",
      "working on 2848\n",
      "working on 2849\n",
      "working on 2850\n",
      "working on 2851\n",
      "working on 2852\n",
      "working on 2853\n",
      "working on 2854\n",
      "working on 2855\n",
      "working on 2856\n",
      "working on 2857\n",
      "working on 2858\n",
      "working on 2859\n",
      "working on 2860\n",
      "working on 2861\n",
      "working on 2862\n",
      "working on 2863\n",
      "working on 2864\n",
      "working on 2865\n",
      "working on 2866\n",
      "working on 2867\n",
      "working on 2868\n",
      "working on 2869\n",
      "working on 2870\n",
      "working on 2871\n",
      "working on 2872\n",
      "working on 2873\n",
      "working on 2874\n",
      "working on 2875\n",
      "working on 2876\n",
      "working on 2877\n",
      "working on 2878\n",
      "working on 2879\n",
      "working on 2880\n",
      "working on 2881\n",
      "working on 2882\n",
      "working on 2883\n",
      "working on 2884\n",
      "working on 2885\n",
      "working on 2886\n",
      "working on 2887\n",
      "working on 2888\n",
      "working on 2889\n",
      "working on 2890\n",
      "working on 2891\n",
      "working on 2892\n",
      "working on 2893\n",
      "working on 2894\n",
      "working on 2895\n",
      "working on 2896\n",
      "working on 2897\n",
      "working on 2898\n",
      "working on 2899\n",
      "working on 2900\n",
      "working on 2901\n",
      "working on 2902\n",
      "working on 2903\n",
      "working on 2904\n",
      "working on 2905\n",
      "working on 2906\n",
      "working on 2907\n",
      "working on 2908\n",
      "working on 2909\n",
      "working on 2910\n",
      "working on 2911\n",
      "working on 2912\n",
      "working on 2913\n",
      "working on 2914\n",
      "working on 2915\n",
      "working on 2916\n",
      "working on 2917\n",
      "working on 2918\n",
      "working on 2919\n",
      "working on 2920\n",
      "working on 2921\n",
      "working on 2922\n",
      "working on 2923\n",
      "working on 2924\n",
      "working on 2925\n",
      "working on 2926\n",
      "working on 2927\n",
      "working on 2928\n",
      "working on 2929\n",
      "working on 2930\n",
      "working on 2931\n",
      "working on 2932\n",
      "working on 2933\n",
      "working on 2934\n",
      "working on 2935\n",
      "working on 2936\n",
      "working on 2937\n",
      "working on 2938\n",
      "working on 2939\n",
      "working on 2940\n",
      "working on 2941\n",
      "working on 2942\n",
      "working on 2943\n",
      "working on 2944\n",
      "working on 2945\n",
      "working on 2946\n",
      "working on 2947\n",
      "working on 2948\n",
      "working on 2949\n",
      "working on 2950\n",
      "working on 2951\n",
      "working on 2952\n",
      "working on 2953\n",
      "working on 2954\n",
      "working on 2955\n",
      "working on 2956\n",
      "working on 2957\n",
      "working on 2958\n",
      "working on 2959\n",
      "working on 2960\n",
      "working on 2961\n",
      "working on 2962\n",
      "working on 2963\n",
      "working on 2964\n",
      "working on 2965\n",
      "working on 2966\n",
      "working on 2967\n",
      "working on 2968\n",
      "working on 2969\n",
      "working on 2970\n",
      "working on 2971\n",
      "working on 2972\n",
      "working on 2973\n",
      "working on 2974\n",
      "working on 2975\n",
      "working on 2976\n",
      "working on 2977\n",
      "working on 2978\n",
      "working on 2979\n",
      "working on 2980\n",
      "working on 2981\n",
      "working on 2982\n",
      "working on 2983\n",
      "working on 2984\n",
      "working on 2985\n",
      "working on 2986\n",
      "working on 2987\n",
      "working on 2988\n",
      "working on 2989\n",
      "working on 2990\n",
      "working on 2991\n",
      "working on 2992\n",
      "working on 2993\n",
      "working on 2994\n",
      "working on 2995\n",
      "working on 2996\n",
      "working on 2997\n",
      "working on 2998\n",
      "working on 2999\n",
      "working on 3000\n",
      "working on 3001\n",
      "working on 3002\n",
      "working on 3003\n",
      "working on 3004\n",
      "working on 3005\n",
      "working on 3006\n",
      "working on 3007\n",
      "working on 3008\n",
      "working on 3009\n",
      "working on 3010\n",
      "working on 3011\n",
      "working on 3012\n",
      "working on 3013\n",
      "working on 3014\n",
      "working on 3015\n",
      "working on 3016\n",
      "working on 3017\n",
      "working on 3018\n",
      "working on 3019\n",
      "working on 3020\n",
      "working on 3021\n",
      "working on 3022\n",
      "working on 3023\n",
      "working on 3024\n",
      "working on 3025\n",
      "working on 3026\n",
      "working on 3027\n",
      "working on 3028\n",
      "working on 3029\n",
      "working on 3030\n",
      "working on 3031\n",
      "working on 3032\n",
      "working on 3033\n",
      "working on 3034\n",
      "working on 3035\n",
      "working on 3036\n",
      "working on 3037\n",
      "working on 3038\n",
      "working on 3039\n",
      "working on 3040\n",
      "working on 3041\n",
      "working on 3042\n",
      "working on 3043\n",
      "working on 3044\n",
      "working on 3045\n",
      "working on 3046\n",
      "working on 3047\n",
      "working on 3048\n",
      "working on 3049\n",
      "working on 3050\n",
      "working on 3051\n",
      "working on 3052\n",
      "working on 3053\n",
      "working on 3054\n",
      "working on 3055\n",
      "working on 3056\n",
      "working on 3057\n",
      "working on 3058\n",
      "working on 3059\n",
      "working on 3060\n",
      "working on 3061\n",
      "working on 3062\n",
      "working on 3063\n",
      "working on 3064\n",
      "working on 3065\n",
      "working on 3066\n",
      "working on 3067\n",
      "working on 3068\n",
      "working on 3069\n",
      "working on 3070\n",
      "working on 3071\n",
      "working on 3072\n",
      "working on 3073\n",
      "working on 3074\n",
      "working on 3075\n",
      "working on 3076\n",
      "working on 3077\n",
      "working on 3078\n",
      "working on 3079\n",
      "working on 3080\n",
      "working on 3081\n",
      "working on 3082\n",
      "working on 3083\n",
      "working on 3084\n",
      "working on 3085\n",
      "working on 3086\n",
      "working on 3087\n",
      "working on 3088\n",
      "working on 3089\n",
      "working on 3090\n",
      "working on 3091\n",
      "working on 3092\n",
      "working on 3093\n",
      "working on 3094\n",
      "working on 3095\n",
      "working on 3096\n",
      "working on 3097\n",
      "working on 3098\n",
      "working on 3099\n",
      "working on 3100\n",
      "working on 3101\n",
      "working on 3102\n",
      "working on 3103\n",
      "working on 3104\n",
      "working on 3105\n",
      "working on 3106\n",
      "working on 3107\n",
      "working on 3108\n",
      "working on 3109\n",
      "working on 3110\n",
      "working on 3111\n",
      "working on 3112\n",
      "working on 3113\n",
      "working on 3114\n",
      "working on 3115\n",
      "working on 3116\n",
      "working on 3117\n",
      "working on 3118\n",
      "working on 3119\n",
      "working on 3120\n",
      "working on 3121\n",
      "working on 3122\n",
      "working on 3123\n",
      "working on 3124\n",
      "working on 3125\n",
      "working on 3126\n",
      "working on 3127\n",
      "working on 3128\n",
      "working on 3129\n",
      "working on 3130\n",
      "working on 3131\n",
      "working on 3132\n",
      "working on 3133\n",
      "working on 3134\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "working on 3135\n",
      "working on 3136\n",
      "working on 3137\n",
      "working on 3138\n",
      "working on 3139\n",
      "working on 3140\n",
      "working on 3141\n",
      "working on 3142\n",
      "working on 3143\n",
      "working on 3144\n",
      "working on 3145\n",
      "working on 3146\n",
      "working on 3147\n",
      "working on 3148\n",
      "working on 3149\n",
      "working on 3150\n",
      "working on 3151\n",
      "working on 3152\n",
      "working on 3153\n",
      "working on 3154\n",
      "working on 3155\n",
      "working on 3156\n",
      "working on 3157\n",
      "working on 3158\n",
      "working on 3159\n",
      "working on 3160\n",
      "working on 3161\n",
      "working on 3162\n",
      "working on 3163\n",
      "working on 3164\n",
      "working on 3165\n",
      "working on 3166\n",
      "working on 3167\n",
      "working on 3168\n",
      "working on 3169\n",
      "working on 3170\n",
      "working on 3171\n",
      "working on 3172\n",
      "working on 3173\n",
      "working on 3174\n",
      "working on 3175\n",
      "working on 3176\n",
      "working on 3177\n",
      "working on 3178\n",
      "working on 3179\n",
      "working on 3180\n",
      "working on 3181\n",
      "working on 3182\n",
      "working on 3183\n",
      "working on 3184\n",
      "working on 3185\n",
      "working on 3186\n",
      "working on 3187\n",
      "working on 3188\n",
      "working on 3189\n",
      "working on 3190\n",
      "working on 3191\n",
      "working on 3192\n",
      "working on 3193\n",
      "working on 3194\n",
      "working on 3195\n",
      "working on 3196\n",
      "working on 3197\n",
      "working on 3198\n",
      "working on 3199\n",
      "working on 3200\n",
      "working on 3201\n",
      "working on 3202\n",
      "working on 3203\n",
      "working on 3204\n",
      "working on 3205\n",
      "working on 3206\n",
      "working on 3207\n",
      "working on 3208\n",
      "working on 3209\n",
      "working on 3210\n",
      "working on 3211\n",
      "working on 3212\n",
      "working on 3213\n",
      "working on 3214\n",
      "working on 3215\n",
      "working on 3216\n",
      "working on 3217\n",
      "working on 3218\n",
      "working on 3219\n",
      "working on 3220\n",
      "working on 3221\n",
      "working on 3222\n",
      "working on 3223\n",
      "working on 3224\n",
      "working on 3225\n",
      "working on 3226\n",
      "working on 3227\n",
      "working on 3228\n",
      "working on 3229\n",
      "working on 3230\n",
      "working on 3231\n",
      "working on 3232\n",
      "working on 3233\n",
      "working on 3234\n",
      "working on 3235\n",
      "working on 3236\n",
      "working on 3237\n",
      "working on 3238\n",
      "working on 3239\n",
      "working on 3240\n",
      "working on 3241\n",
      "working on 3242\n",
      "working on 3243\n",
      "working on 3244\n",
      "working on 3245\n",
      "working on 3246\n",
      "working on 3247\n",
      "working on 3248\n",
      "working on 3249\n",
      "working on 3250\n",
      "working on 3251\n",
      "working on 3252\n",
      "working on 3253\n",
      "working on 3254\n",
      "working on 3255\n",
      "working on 3256\n",
      "working on 3257\n",
      "working on 3258\n",
      "working on 3259\n",
      "working on 3260\n",
      "working on 3261\n",
      "working on 3262\n",
      "working on 3263\n",
      "working on 3264\n",
      "working on 3265\n",
      "working on 3266\n",
      "working on 3267\n",
      "working on 3268\n",
      "working on 3269\n",
      "working on 3270\n",
      "working on 3271\n",
      "working on 3272\n",
      "working on 3273\n",
      "working on 3274\n",
      "working on 3275\n",
      "working on 3276\n",
      "working on 3277\n",
      "working on 3278\n",
      "working on 3279\n",
      "working on 3280\n",
      "working on 3281\n",
      "working on 3282\n",
      "working on 3283\n",
      "working on 3284\n",
      "working on 3285\n",
      "working on 3286\n",
      "working on 3287\n",
      "working on 3288\n",
      "working on 3289\n",
      "working on 3290\n",
      "working on 3291\n",
      "working on 3292\n",
      "working on 3293\n",
      "working on 3294\n",
      "working on 3295\n",
      "working on 3296\n",
      "working on 3297\n",
      "working on 3298\n",
      "working on 3299\n",
      "working on 3300\n",
      "working on 3301\n",
      "working on 3302\n",
      "working on 3303\n",
      "working on 3304\n",
      "working on 3305\n",
      "working on 3306\n",
      "working on 3307\n",
      "working on 3308\n",
      "working on 3309\n",
      "working on 3310\n",
      "working on 3311\n",
      "working on 3312\n",
      "working on 3313\n",
      "working on 3314\n",
      "working on 3315\n",
      "working on 3316\n",
      "working on 3317\n",
      "working on 3318\n",
      "working on 3319\n",
      "working on 3320\n",
      "working on 3321\n",
      "working on 3322\n",
      "working on 3323\n",
      "working on 3324\n",
      "working on 3325\n",
      "working on 3326\n",
      "working on 3327\n",
      "working on 3328\n",
      "working on 3329\n",
      "working on 3330\n",
      "working on 3331\n",
      "working on 3332\n",
      "working on 3333\n",
      "working on 3334\n",
      "working on 3335\n",
      "working on 3336\n",
      "working on 3337\n",
      "working on 3338\n",
      "working on 3339\n",
      "working on 3340\n",
      "working on 3341\n",
      "working on 3342\n",
      "working on 3343\n",
      "working on 3344\n",
      "working on 3345\n",
      "working on 3346\n",
      "working on 3347\n",
      "working on 3348\n",
      "working on 3349\n",
      "working on 3350\n",
      "working on 3351\n",
      "working on 3352\n",
      "working on 3353\n",
      "working on 3354\n",
      "working on 3355\n",
      "working on 3356\n",
      "working on 3357\n",
      "working on 3358\n",
      "working on 3359\n",
      "working on 3360\n",
      "working on 3361\n",
      "working on 3362\n",
      "working on 3363\n",
      "working on 3364\n",
      "working on 3365\n",
      "working on 3366\n",
      "working on 3367\n",
      "working on 3368\n",
      "working on 3369\n",
      "working on 3370\n",
      "working on 3371\n",
      "working on 3372\n",
      "working on 3373\n",
      "working on 3374\n",
      "working on 3375\n",
      "working on 3376\n",
      "working on 3377\n",
      "working on 3378\n",
      "working on 3379\n",
      "working on 3380\n",
      "working on 3381\n",
      "working on 3382\n",
      "working on 3383\n",
      "working on 3384\n",
      "working on 3385\n",
      "working on 3386\n",
      "working on 3387\n",
      "working on 3388\n",
      "working on 3389\n",
      "working on 3390\n",
      "working on 3391\n",
      "working on 3392\n",
      "working on 3393\n",
      "working on 3394\n",
      "working on 3395\n",
      "working on 3396\n",
      "working on 3397\n",
      "working on 3398\n",
      "working on 3399\n",
      "working on 3400\n",
      "working on 3401\n",
      "working on 3402\n",
      "working on 3403\n",
      "working on 3404\n",
      "working on 3405\n",
      "working on 3406\n",
      "working on 3407\n",
      "working on 3408\n",
      "working on 3409\n",
      "working on 3410\n",
      "working on 3411\n",
      "working on 3412\n",
      "working on 3413\n",
      "working on 3414\n",
      "working on 3415\n",
      "working on 3416\n",
      "working on 3417\n",
      "working on 3418\n",
      "working on 3419\n",
      "working on 3420\n",
      "working on 3421\n",
      "working on 3422\n",
      "working on 3423\n",
      "working on 3424\n",
      "working on 3425\n",
      "working on 3426\n",
      "working on 3427\n",
      "working on 3428\n",
      "working on 3429\n",
      "working on 3430\n",
      "working on 3431\n",
      "working on 3432\n",
      "working on 3433\n",
      "working on 3434\n",
      "working on 3435\n",
      "working on 3436\n",
      "working on 3437\n",
      "working on 3438\n",
      "working on 3439\n",
      "working on 3440\n",
      "working on 3441\n",
      "working on 3442\n",
      "working on 3443\n",
      "working on 3444\n",
      "working on 3445\n",
      "working on 3446\n",
      "working on 3447\n",
      "working on 3448\n",
      "working on 3449\n",
      "working on 3450\n",
      "working on 3451\n",
      "working on 3452\n",
      "working on 3453\n",
      "working on 3454\n",
      "working on 3455\n",
      "working on 3456\n",
      "working on 3457\n",
      "working on 3458\n",
      "working on 3459\n",
      "working on 3460\n",
      "working on 3461\n",
      "working on 3462\n",
      "working on 3463\n",
      "working on 3464\n",
      "working on 3465\n",
      "working on 3466\n",
      "working on 3467\n",
      "working on 3468\n",
      "working on 3469\n",
      "working on 3470\n",
      "working on 3471\n",
      "working on 3472\n",
      "working on 3473\n",
      "working on 3474\n",
      "working on 3475\n",
      "working on 3476\n",
      "working on 3477\n",
      "working on 3478\n",
      "working on 3479\n",
      "working on 3480\n",
      "working on 3481\n",
      "working on 3482\n",
      "working on 3483\n",
      "working on 3484\n",
      "working on 3485\n",
      "working on 3486\n",
      "working on 3487\n",
      "working on 3488\n",
      "working on 3489\n",
      "working on 3490\n",
      "working on 3491\n",
      "working on 3492\n",
      "working on 3493\n",
      "working on 3494\n",
      "working on 3495\n",
      "working on 3496\n",
      "working on 3497\n",
      "working on 3498\n",
      "working on 3499\n",
      "working on 3500\n",
      "working on 3501\n",
      "working on 3502\n",
      "working on 3503\n",
      "working on 3504\n",
      "working on 3505\n",
      "working on 3506\n",
      "working on 3507\n",
      "working on 3508\n",
      "working on 3509\n",
      "working on 3510\n",
      "working on 3511\n",
      "working on 3512\n",
      "working on 3513\n",
      "working on 3514\n",
      "working on 3515\n",
      "working on 3516\n",
      "working on 3517\n",
      "working on 3518\n",
      "working on 3519\n",
      "working on 3520\n",
      "working on 3521\n",
      "working on 3522\n",
      "working on 3523\n",
      "working on 3524\n",
      "working on 3525\n",
      "working on 3526\n",
      "working on 3527\n",
      "working on 3528\n",
      "working on 3529\n",
      "working on 3530\n",
      "working on 3531\n",
      "working on 3532\n",
      "working on 3533\n",
      "working on 3534\n",
      "working on 3535\n",
      "working on 3536\n",
      "working on 3537\n",
      "working on 3538\n",
      "working on 3539\n",
      "working on 3540\n",
      "working on 3541\n",
      "working on 3542\n",
      "working on 3543\n",
      "working on 3544\n",
      "working on 3545\n",
      "working on 3546\n",
      "working on 3547\n",
      "working on 3548\n",
      "working on 3549\n",
      "working on 3550\n",
      "working on 3551\n",
      "working on 3552\n",
      "working on 3553\n",
      "working on 3554\n",
      "working on 3555\n",
      "working on 3556\n",
      "working on 3557\n",
      "working on 3558\n",
      "working on 3559\n",
      "working on 3560\n",
      "working on 3561\n",
      "working on 3562\n",
      "working on 3563\n",
      "working on 3564\n",
      "working on 3565\n",
      "working on 3566\n",
      "working on 3567\n",
      "working on 3568\n",
      "working on 3569\n",
      "working on 3570\n",
      "working on 3571\n",
      "working on 3572\n",
      "working on 3573\n",
      "working on 3574\n",
      "working on 3575\n",
      "working on 3576\n",
      "working on 3577\n",
      "working on 3578\n",
      "working on 3579\n",
      "working on 3580\n",
      "working on 3581\n",
      "working on 3582\n",
      "working on 3583\n",
      "working on 3584\n",
      "working on 3585\n",
      "working on 3586\n",
      "working on 3587\n",
      "working on 3588\n",
      "working on 3589\n",
      "working on 3590\n",
      "working on 3591\n",
      "working on 3592\n",
      "working on 3593\n",
      "working on 3594\n",
      "working on 3595\n",
      "working on 3596\n",
      "working on 3597\n",
      "working on 3598\n",
      "working on 3599\n",
      "working on 3600\n",
      "working on 3601\n",
      "working on 3602\n",
      "working on 3603\n",
      "working on 3604\n",
      "working on 3605\n",
      "working on 3606\n",
      "working on 3607\n",
      "working on 3608\n",
      "working on 3609\n",
      "working on 3610\n",
      "working on 3611\n",
      "working on 3612\n",
      "working on 3613\n",
      "working on 3614\n",
      "working on 3615\n",
      "working on 3616\n",
      "working on 3617\n",
      "working on 3618\n",
      "working on 3619\n",
      "working on 3620\n",
      "working on 3621\n",
      "working on 3622\n",
      "working on 3623\n",
      "working on 3624\n",
      "working on 3625\n",
      "working on 3626\n",
      "working on 3627\n",
      "working on 3628\n",
      "working on 3629\n",
      "working on 3630\n",
      "working on 3631\n",
      "working on 3632\n",
      "working on 3633\n",
      "working on 3634\n",
      "working on 3635\n",
      "working on 3636\n",
      "working on 3637\n",
      "working on 3638\n",
      "working on 3639\n",
      "working on 3640\n",
      "working on 3641\n",
      "working on 3642\n",
      "working on 3643\n",
      "working on 3644\n",
      "working on 3645\n",
      "working on 3646\n",
      "working on 3647\n",
      "working on 3648\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "working on 3649\n",
      "working on 3650\n",
      "working on 3651\n",
      "working on 3652\n",
      "working on 3653\n",
      "working on 3654\n",
      "working on 3655\n",
      "working on 3656\n",
      "working on 3657\n",
      "working on 3658\n",
      "working on 3659\n",
      "working on 3660\n",
      "working on 3661\n",
      "working on 3662\n",
      "working on 3663\n",
      "working on 3664\n",
      "working on 3665\n",
      "working on 3666\n",
      "working on 3667\n",
      "working on 3668\n",
      "working on 3669\n",
      "working on 3670\n",
      "working on 3671\n",
      "working on 3672\n",
      "working on 3673\n",
      "working on 3674\n",
      "working on 3675\n",
      "working on 3676\n",
      "working on 3677\n",
      "working on 3678\n",
      "working on 3679\n",
      "working on 3680\n",
      "working on 3681\n",
      "working on 3682\n",
      "working on 3683\n",
      "working on 3684\n",
      "working on 3685\n",
      "working on 3686\n",
      "working on 3687\n",
      "working on 3688\n",
      "working on 3689\n",
      "working on 3690\n",
      "working on 3691\n",
      "working on 3692\n",
      "working on 3693\n",
      "working on 3694\n",
      "working on 3695\n",
      "working on 3696\n",
      "working on 3697\n",
      "working on 3698\n",
      "working on 3699\n",
      "working on 3700\n",
      "working on 3701\n",
      "working on 3702\n",
      "working on 3703\n",
      "working on 3704\n",
      "working on 3705\n",
      "working on 3706\n",
      "working on 3707\n",
      "working on 3708\n",
      "working on 3709\n",
      "working on 3710\n",
      "working on 3711\n",
      "working on 3712\n",
      "working on 3713\n",
      "working on 3714\n",
      "working on 3715\n",
      "working on 3716\n",
      "working on 3717\n",
      "working on 3718\n",
      "working on 3719\n",
      "working on 3720\n",
      "working on 3721\n",
      "working on 3722\n",
      "working on 3723\n",
      "working on 3724\n",
      "working on 3725\n",
      "working on 3726\n",
      "working on 3727\n",
      "working on 3728\n",
      "working on 3729\n",
      "working on 3730\n",
      "working on 3731\n",
      "working on 3732\n",
      "working on 3733\n",
      "working on 3734\n",
      "working on 3735\n",
      "working on 3736\n",
      "working on 3737\n",
      "working on 3738\n",
      "working on 3739\n",
      "working on 3740\n",
      "working on 3741\n",
      "working on 3742\n",
      "working on 3743\n",
      "working on 3744\n",
      "working on 3745\n",
      "working on 3746\n",
      "working on 3747\n",
      "working on 3748\n",
      "working on 3749\n",
      "working on 3750\n",
      "working on 3751\n",
      "working on 3752\n",
      "working on 3753\n",
      "working on 3754\n",
      "working on 3755\n",
      "working on 3756\n",
      "working on 3757\n",
      "working on 3758\n",
      "working on 3759\n",
      "working on 3760\n",
      "working on 3761\n",
      "working on 3762\n",
      "working on 3763\n",
      "working on 3764\n",
      "working on 3765\n",
      "working on 3766\n",
      "working on 3767\n",
      "working on 3768\n",
      "working on 3769\n",
      "working on 3770\n",
      "working on 3771\n",
      "working on 3772\n",
      "working on 3773\n",
      "working on 3774\n",
      "working on 3775\n",
      "working on 3776\n",
      "working on 3777\n",
      "working on 3778\n",
      "working on 3779\n",
      "working on 3780\n",
      "working on 3781\n",
      "working on 3782\n",
      "working on 3783\n",
      "working on 3784\n",
      "working on 3785\n",
      "working on 3786\n",
      "working on 3787\n",
      "working on 3788\n",
      "working on 3789\n",
      "working on 3790\n",
      "working on 3791\n",
      "working on 3792\n",
      "working on 3793\n",
      "working on 3794\n",
      "working on 3795\n",
      "working on 3796\n",
      "working on 3797\n",
      "working on 3798\n",
      "working on 3799\n",
      "working on 3800\n",
      "working on 3801\n",
      "working on 3802\n",
      "working on 3803\n",
      "working on 3804\n",
      "working on 3805\n",
      "working on 3806\n",
      "working on 3807\n",
      "working on 3808\n",
      "working on 3809\n",
      "working on 3810\n",
      "working on 3811\n",
      "working on 3812\n",
      "working on 3813\n",
      "working on 3814\n",
      "working on 3815\n",
      "working on 3816\n",
      "working on 3817\n",
      "working on 3818\n",
      "working on 3819\n",
      "working on 3820\n",
      "working on 3821\n",
      "working on 3822\n",
      "working on 3823\n",
      "working on 3824\n",
      "working on 3825\n",
      "working on 3826\n",
      "working on 3827\n",
      "working on 3828\n",
      "working on 3829\n",
      "working on 3830\n",
      "working on 3831\n",
      "working on 3832\n",
      "working on 3833\n",
      "working on 3834\n",
      "working on 3835\n",
      "working on 3836\n",
      "working on 3837\n",
      "working on 3838\n",
      "working on 3839\n",
      "working on 3840\n",
      "working on 3841\n",
      "working on 3842\n",
      "working on 3843\n",
      "working on 3844\n",
      "working on 3845\n",
      "working on 3846\n",
      "working on 3847\n",
      "working on 3848\n",
      "working on 3849\n",
      "working on 3850\n",
      "working on 3851\n",
      "working on 3852\n",
      "working on 3853\n",
      "working on 3854\n",
      "working on 3855\n",
      "working on 3856\n",
      "working on 3857\n",
      "working on 3858\n",
      "working on 3859\n",
      "working on 3860\n",
      "working on 3861\n",
      "working on 3862\n",
      "working on 3863\n",
      "working on 3864\n",
      "working on 3865\n",
      "working on 3866\n",
      "working on 3867\n",
      "working on 3868\n",
      "working on 3869\n",
      "working on 3870\n",
      "working on 3871\n",
      "working on 3872\n",
      "working on 3873\n",
      "working on 3874\n",
      "working on 3875\n",
      "working on 3876\n",
      "working on 3877\n",
      "working on 3878\n",
      "working on 3879\n",
      "working on 3880\n",
      "working on 3881\n",
      "working on 3882\n",
      "working on 3883\n",
      "working on 3884\n",
      "working on 3885\n",
      "working on 3886\n",
      "working on 3887\n",
      "working on 3888\n",
      "working on 3889\n",
      "working on 3890\n",
      "working on 3891\n",
      "working on 3892\n",
      "working on 3893\n",
      "working on 3894\n",
      "working on 3895\n",
      "working on 3896\n",
      "working on 3897\n",
      "working on 3898\n",
      "working on 3899\n",
      "working on 3900\n",
      "working on 3901\n",
      "working on 3902\n",
      "working on 3903\n",
      "working on 3904\n",
      "working on 3905\n",
      "working on 3906\n",
      "working on 3907\n",
      "working on 3908\n",
      "working on 3909\n",
      "working on 3910\n",
      "working on 3911\n",
      "working on 3912\n",
      "working on 3913\n",
      "working on 3914\n",
      "working on 3915\n",
      "working on 3916\n",
      "working on 3917\n",
      "working on 3918\n",
      "working on 3919\n",
      "working on 3920\n",
      "working on 3921\n",
      "working on 3922\n",
      "working on 3923\n",
      "working on 3924\n",
      "working on 3925\n",
      "working on 3926\n",
      "working on 3927\n",
      "working on 3928\n",
      "working on 3929\n",
      "working on 3930\n",
      "working on 3931\n",
      "working on 3932\n",
      "working on 3933\n",
      "working on 3934\n",
      "working on 3935\n",
      "working on 3936\n",
      "working on 3937\n",
      "working on 3938\n",
      "working on 3939\n",
      "working on 3940\n",
      "working on 3941\n",
      "working on 3942\n",
      "working on 3943\n",
      "working on 3944\n",
      "working on 3945\n",
      "working on 3946\n",
      "working on 3947\n",
      "working on 3948\n",
      "working on 3949\n",
      "working on 3950\n",
      "working on 3951\n",
      "working on 3952\n",
      "working on 3953\n",
      "working on 3954\n",
      "working on 3955\n",
      "working on 3956\n",
      "working on 3957\n",
      "working on 3958\n",
      "working on 3959\n",
      "working on 3960\n",
      "working on 3961\n",
      "working on 3962\n",
      "working on 3963\n",
      "working on 3964\n",
      "working on 3965\n",
      "working on 3966\n",
      "working on 3967\n",
      "working on 3968\n",
      "working on 3969\n",
      "working on 3970\n",
      "working on 3971\n",
      "working on 3972\n",
      "working on 3973\n",
      "working on 3974\n",
      "working on 3975\n",
      "working on 3976\n",
      "working on 3977\n",
      "working on 3978\n",
      "working on 3979\n",
      "working on 3980\n",
      "working on 3981\n",
      "working on 3982\n",
      "working on 3983\n",
      "working on 3984\n",
      "working on 3985\n",
      "working on 3986\n",
      "working on 3987\n",
      "working on 3988\n",
      "working on 3989\n",
      "working on 3990\n",
      "working on 3991\n",
      "working on 3992\n",
      "working on 3993\n",
      "working on 3994\n",
      "working on 3995\n",
      "working on 3996\n",
      "working on 3997\n",
      "working on 3998\n",
      "working on 3999\n",
      "working on 4000\n",
      "working on 4001\n",
      "working on 4002\n",
      "working on 4003\n",
      "working on 4004\n",
      "working on 4005\n",
      "working on 4006\n",
      "working on 4007\n",
      "working on 4008\n",
      "working on 4009\n",
      "working on 4010\n",
      "working on 4011\n",
      "working on 4012\n",
      "working on 4013\n",
      "working on 4014\n",
      "working on 4015\n",
      "working on 4016\n",
      "working on 4017\n",
      "working on 4018\n",
      "working on 4019\n",
      "working on 4020\n",
      "working on 4021\n",
      "working on 4022\n",
      "working on 4023\n",
      "working on 4024\n",
      "working on 4025\n",
      "working on 4026\n",
      "working on 4027\n",
      "working on 4028\n",
      "working on 4029\n",
      "working on 4030\n",
      "working on 4031\n",
      "working on 4032\n",
      "working on 4033\n",
      "working on 4034\n",
      "working on 4035\n",
      "working on 4036\n",
      "working on 4037\n",
      "working on 4038\n",
      "working on 4039\n",
      "working on 4040\n",
      "working on 4041\n",
      "working on 4042\n",
      "working on 4043\n",
      "working on 4044\n",
      "working on 4045\n",
      "working on 4046\n",
      "working on 4047\n",
      "working on 4048\n",
      "working on 4049\n",
      "working on 4050\n",
      "working on 4051\n",
      "working on 4052\n",
      "working on 4053\n",
      "working on 4054\n",
      "working on 4055\n",
      "working on 4056\n",
      "working on 4057\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "class InferenceConfig(ChipsConfig):\n",
    "    GPU_COUNT = 1\n",
    "    IMAGES_PER_GPU = 1\n",
    "    \n",
    "inference_config = InferenceConfig()\n",
    "\n",
    "# MODEL_DIR = os.path.join(ROOT_DIR, \"logs_chips3\")\n",
    "model_path = MODEL_DIR+'/chips20181016T1457/mask_rcnn_chips_0012.h5' \n",
    "#model_path = 'mask_rcnn_chips_3_0905_null_train.h5'\n",
    "# Recreate the model in inference mode\n",
    "model = modellib.MaskRCNN(mode=\"inference\", \n",
    "                          config=inference_config,\n",
    "                          model_dir= MODEL_DIR)\n",
    "\n",
    "# Get path to saved weights\n",
    "# Either set a specific path or find last trained weights\n",
    "#model_path = os.path.join(MODEL_DIR, \"mask_rcnn_chips.h5\")\n",
    "#model_path = model.find_last()\n",
    "\n",
    "# Load trained weights\n",
    "print(\"Loading weights from \", model_path)\n",
    "model.load_weights(model_path, by_name=True)\n",
    "\n",
    "import pandas as pd\n",
    "import time \n",
    "\n",
    "def model_eval(dataset,name):\n",
    "    start = time.time()\n",
    "    if name == 'train':\n",
    "        image_ids = dataset.image_ids\n",
    "    else:\n",
    "        image_ids = dataset.image_ids\n",
    "        \n",
    "    gt_class = []\n",
    "    pre_class = []\n",
    "    ap = []\n",
    "    score = []\n",
    "    \n",
    "    for image_id in image_ids:\n",
    "        print('working on %d'%image_id)\n",
    "        # Load image and ground truth data\n",
    "        image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n",
    "            modellib.load_image_gt(dataset, inference_config,\n",
    "                                   image_id, use_mini_mask=False)\n",
    "        # molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)\n",
    "        # Run object detection\n",
    "        results = model.detect([image], verbose=0)\n",
    "        r = results[0]\n",
    "\n",
    "        # Compute AP\n",
    "        if len(r['class_ids'])>0 and len(gt_class_id)>0 :\n",
    "            AP, precisions, recalls, overlaps =utils.compute_ap(gt_bbox, gt_class_id, gt_mask,\n",
    "                                 r[\"rois\"], r[\"class_ids\"], r[\"scores\"], r['masks'],iou_threshold=0.5)\n",
    "            #print(AP)\n",
    "            ap.append(AP)\n",
    "        else:\n",
    "            ap.append(0)\n",
    "\n",
    "        if not len(r['class_ids']) :\n",
    "            pre_class.append(0)\n",
    "            score.append(0)\n",
    "        else:\n",
    "            pre_class.append(r['class_ids'][0])\n",
    "            score.append(r['scores'][0])\n",
    "\n",
    "        if not len(gt_class_id):\n",
    "            gt_class.append(0)\n",
    "        else:\n",
    "            gt_class.append(gt_class_id[0])\n",
    "    \n",
    "   \n",
    "    # predict summary\n",
    "    result = pd.DataFrame()\n",
    "    result['image_id'] = image_ids\n",
    "    result['gt_class'] = gt_class\n",
    "    result['pre_class'] = pre_class\n",
    "    result['ap'] = ap\n",
    "    result['score']=score\n",
    "    \n",
    "    return result \n",
    "\n",
    "def info_display(result):\n",
    "    \n",
    "    image_ids = result['image_id']\n",
    "    gt_class = result['gt_class']\n",
    "    pre_class = result['pre_class']\n",
    "    ap = result['ap']\n",
    "    \n",
    "    #result display\n",
    "    diff = np.array(gt_class)-np.array(pre_class)\n",
    "    diff = diff.tolist()\n",
    "    precision = diff.count(0)/len(diff)\n",
    "    total_false = len(diff)-diff.count(0)\n",
    "    \n",
    "    negative = len(result[result['gt_class']==0])\n",
    "    positive = len(result)-negative\n",
    "    \n",
    "    false_positive = len(result[result['gt_class']==0][result['pre_class']!=0])\n",
    "    false_negative = len(result[result['gt_class']!=0][result['pre_class']==0])\n",
    "    \n",
    "    #print('evaluation on %s'%name)\n",
    "    print('mAP = ',np.mean(ap))\n",
    "    print('precision =  %d%%'%(precision*100))\n",
    "    print('false_positive/total = %d%%'%(false_positive/len(image_ids)*100))\n",
    "    print('false_negative/toal = %d%%'%(false_negative/len(image_ids)*100))\n",
    "  \n",
    "    print('false_positive/negative = %d%%'%((false_positive/negative)*100))\n",
    "    print('false_negative/positive = %d%%'%((false_negative/positive)*100))\n",
    "    \n",
    "test_result = model_eval(dataset_test,'test')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mAP =  0.626642845408247\n",
      "precision =  80%\n",
      "false_positive/total = 4%\n",
      "false_negative/toal = 5%\n",
      "false_positive/negative = 25%\n",
      "false_negative/positive = 6%\n"
     ]
    }
   ],
   "source": [
    "info_display(test_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix as cm\n",
    "def error_display(result, num=1):\n",
    "    y_true = result['gt_class']\n",
    "    y_pred = result['pre_class']\n",
    "    cmatrix = cm(y_true, y_pred)\n",
    "\n",
    "    num_finechips = sum(cmatrix[0])\n",
    "    num_flawchips = len(result) - num_finechips\n",
    "    num_pre_finechips = sum(cmatrix[:, 0])\n",
    "    num_pre_flawchips = len(result) - num_pre_finechips\n",
    "\n",
    "    print('confusion matrix:\\n')\n",
    "    print(cmatrix)\n",
    "\n",
    "    pres = []\n",
    "    recs = []\n",
    "    for i in range(4):\n",
    "        precison = cmatrix[i, i] / (sum(cmatrix[i]) +1)* 100\n",
    "        recall = cmatrix[i, i] / (sum(cmatrix[:, i])+1) * 100\n",
    "        pres.append(precison)\n",
    "        recs.append(recall)\n",
    "        print('precision and recall on class%d ： %d%%   %d%% \\n' % (i, precison, recall))\n",
    "\n",
    "    print('total validation samples num ：  %d' % len(result))\n",
    "    print('mean presicion ： %d%%' % (np.mean(pres)))\n",
    "    print('mean recall ： %d%%' % (np.mean(recs)))\n",
    "    print('mean ap ：  %d%% ' % (np.mean(result['ap']) * 100))\n",
    "    return\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix:\n",
      "\n",
      "[[ 537  103   71   12]\n",
      " [  97  754   88    7]\n",
      " [  86  112 1161   80]\n",
      " [  25   29   72  824]]\n",
      "precision and recall on class0 ： 74%   71% \n",
      "\n",
      "precision and recall on class1 ： 79%   75% \n",
      "\n",
      "precision and recall on class2 ： 80%   83% \n",
      "\n",
      "precision and recall on class3 ： 86%   89% \n",
      "\n",
      "total validation samples num ：  4058\n",
      "mean presicion ： 80%\n",
      "mean recall ： 79%\n",
      "mean ap ：  62% \n"
     ]
    }
   ],
   "source": [
    "error_display(test_result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### model eval at epoch 5 (test_chips3_15.csv)\n",
    "- mAP =  0.7668939776617837\n",
    "- precision =  96%\n",
    "- false_positive/total = 1%\n",
    "- false_negative/toal = 1%\n",
    "- false_positive/negative = 6%\n",
    "- false_negative/positive = 1%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_result.to_csv('test_chips3_data4_1015.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
